Add fine-grain ingress DNS control through CRD

Context
---

Handling usual cluster operations, we often come to operate higher risk changes like bumping ingress controller
versions, changing the underneath ingress service type (for example moving from an AWS ELB to an AWS NLB).

Doing so, the safest way would be to be able to provision a new ingress controller and progressively migrating traffic
to the new instance.

Problems
---

Currently, traffic controller reads the host and load balancer reference using
the ingress status.

This prevents from being able to handle and control weighted records across the different ingress controllers.

Goal
---

Enable fine-grained routing between various ingress controllers in the same cluster.

Unblocked use-cases
---

- Progressively change and test the ingress infrastructure (load balancer, ...) and versions
- Allow sharding ingress controllers at the DNS level

Change-Id: I196c8d60d03a6b7560f541eedaf2d8438df39a53

cmd/traffic-controller/main.go

@@ -18,7 +18,7 @@ import (
 )
 
 var (
-	scheme   = controllers.NewScheme()
+	scheme   = controllers.Must(controllers.NewScheme())
 	setupLog = ctrl.Log.WithName("setup")
 )
 

config/crd/bases/ingress.adevinta.com_clusteringressservicednsweights.yaml

@@ -0,0 +1,176 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.4
+  name: clusteringressservicednsweights.ingress.adevinta.com
+spec:
+  group: ingress.adevinta.com
+  names:
+    kind: ClusterIngressServiceDNSWeight
+    listKind: ClusterIngressServiceDNSWeightList
+    plural: clusteringressservicednsweights
+    singular: clusteringressservicednsweight
+  scope: Cluster
+  versions:
+  - name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: ClusterIngressServiceDNSWeight is the defines the links between
+          k8s services and ingresses
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: IngressServiceDNSWeightSpec defines the desired state of
+              IngressServiceDNSWeight
+            properties:
+              identifier:
+                type: string
+              ingressSelector:
+                description: IngressSelector defines the ingress selector in specific
+                  namespaces
+                properties:
+                  classes:
+                    description: |-
+                      Classes is the list of classes that the ingress must have
+                      It must contain at least one class
+                    items:
+                      type: string
+                    type: array
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                  namespaces:
+                    description: |-
+                      Namespaces is the list of namespaces where the ingress selector will be applied
+                      If not provided, all namespaces will be considered
+                    items:
+                      type: string
+                    type: array
+                required:
+                - classes
+                type: object
+                x-kubernetes-map-type: atomic
+              serviceSelector:
+                description: ServiceSelector defines the service selector in a specific
+                  namespace
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                  namespace:
+                    type: string
+                required:
+                - namespace
+                type: object
+                x-kubernetes-map-type: atomic
+              weight:
+                default: 0
+                description: Weight is the weight of the service
+                maximum: 100
+                minimum: 0
+                type: integer
+            required:
+            - identifier
+            - ingressSelector
+            - serviceSelector
+            - weight
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true

pkg/apis/ingress.adevinta.com/v1beta1/groupversion_info.go

@@ -0,0 +1,20 @@
+// Package v1alpha1 contains API Schema definitions for the deployment v1alpha1 API group
+// +kubebuilder:object:generate=true
+// +groupName=ingress.adevinta.com
+package v1beta1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "ingress.adevinta.com", Version: "v1alpha1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)

pkg/apis/ingress.adevinta.com/v1beta1/ingress_types.go

@@ -0,0 +1,83 @@
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// kind: ClusterIngressServiceDNSWeight
+// spec:
+// weight: <int>
+// identifier: <string>
+// serviceSelector:
+//
+//	namespace: <string>
+//	matchLabels:
+//		<string>: <string>
+//
+// ingressSelector:
+//
+//	matchLabels:
+//		<string>: <string>
+//	namespaces:
+//		- <string>
+//	classes:
+//		- <string>
+
+// ClusterIngressServiceDNSWeight is the defines the links between k8s services and ingresses
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:scope=Cluster
+type ClusterIngressServiceDNSWeight struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              ClusterIngressServiceDNSWeightSpec `json:"spec"`
+}
+
+// ClusterIngressServiceDNSWeightList contains a list of ClusterIngressServiceDNSWeight
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:scope=Cluster
+type ClusterIngressServiceDNSWeightList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ClusterIngressServiceDNSWeight `json:"items"`
+}
+
+// IngressServiceDNSWeightSpec defines the desired state of IngressServiceDNSWeight
+type ClusterIngressServiceDNSWeightSpec struct {
+	// Weight is the weight of the service
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:validation:Maximum=100
+	// +kubebuilder:default:=0
+	Weight uint `json:"weight"`
+
+	Identifier      string          `json:"identifier"`
+	ServiceSelector ServiceSelector `json:"serviceSelector"`
+	IngressSelector IngressSelector `json:"ingressSelector"`
+}
+
+// ServiceSelector defines the service selector in a specific namespace
+type ServiceSelector struct {
+	Namespace            string `json:"namespace"`
+	metav1.LabelSelector `json:",inline"`
+}
+
+// IngressSelector defines the ingress selector in specific namespaces
+type IngressSelector struct {
+
+	// LabelSelector is the list of labels that the ingress must have
+	// If not provided, all ingresses will be considered
+	metav1.LabelSelector `json:",inline"`
+
+	// Namespaces is the list of namespaces where the ingress selector will be applied
+	// If not provided, all namespaces will be considered
+	// +kubebuilder:validation:Optional
+	Namespaces []string `json:"namespaces,omitempty"`
+
+	// Classes is the list of classes that the ingress must have
+	// It must contain at least one class
+	// +kubebuilder:validation:minItems=1
+	Classes []string `json:"classes"`
+}
+
+func init() {
+	SchemeBuilder.Register(&ClusterIngressServiceDNSWeight{}, &ClusterIngressServiceDNSWeightList{})
+}

pkg/apis/ingress.adevinta.com/v1beta1/serialization_test.go

@@ -0,0 +1,65 @@
+package v1beta1_test
+
+import (
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubectl/pkg/scheme"
+
+	"github.com/adevinta/k8s-traffic-controller/pkg/apis/ingress.adevinta.com/v1beta1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFullyFlegedDeserialization(t *testing.T) {
+	serialized := `kind: ClusterIngressServiceDNSWeight
+spec:
+  weight: 80
+  identifier: "public-nginx-classic-elb"
+  serviceSelector:
+    namespace: "service-namespace"
+    matchLabels:
+      "some": "value"
+  ingressSelector:
+    matchLabels:
+      "key": "value"
+    namespaces:
+    - "my-namespace"
+    classes:
+    - "public"
+`
+
+	clusteringressweight := v1beta1.ClusterIngressServiceDNSWeight{}
+	object, _, err := scheme.Codecs.UniversalDeserializer().Decode([]byte(serialized), nil, &clusteringressweight)
+	require.NoError(t, err)
+	assert.Equal(
+		t,
+		&v1beta1.ClusterIngressServiceDNSWeight{
+			TypeMeta: metav1.TypeMeta{
+				Kind: "ClusterIngressServiceDNSWeight",
+			},
+			Spec: v1beta1.ClusterIngressServiceDNSWeightSpec{
+				Weight:     80,
+				Identifier: "public-nginx-classic-elb",
+				ServiceSelector: v1beta1.ServiceSelector{
+					Namespace: "service-namespace",
+					LabelSelector: metav1.LabelSelector{
+						MatchLabels: map[string]string{
+							"some": "value",
+						},
+					},
+				},
+				IngressSelector: v1beta1.IngressSelector{
+					LabelSelector: metav1.LabelSelector{
+						MatchLabels: map[string]string{
+							"key": "value",
+						},
+					},
+					Namespaces: []string{"my-namespace"},
+					Classes:    []string{"public"},
+				},
+			},
+		},
+		object,
+	)
+}