feat: Support different namespace for infra resources

smartxworks · May 23, 2023 · 0244207 · 0244207
1 parent 09c3204
commit 0244207
Show file tree

Hide file tree

Showing 12 changed files with 2,721 additions and 2,484 deletions.
diff --git a/README.md b/README.md
@@ -35,6 +35,7 @@ Except for the [common variables](https://cluster-api.sigs.k8s.io/clusterctl/pro
 | KUBERNETES_VERSION                         | Only support Kubernetes versions that corresponding rootfs image `smartxworks/capch-rootfs-$KUBERNETES_VERSION` exists|
 | VIRTINK_INFRA_CLUSTER_SECRET_NAME          | The name of secret in the management cluster that contains the kubeconfig of the Virtink infrastructure cluster       |
 | VIRTINK_INFRA_CLUSTER_SECRET_NAMESPACE     | The namespace of secret in the management cluster that contains the kubeconfig of the Virtink infrastructure cluster  |
+| VIRTINK_INFRA_CLUSTER_RESOURCES_NAMESPACE  | The namespace of resources(such as VM, DataVolume) to be created in the infrastructural cluster, make sure it exists before create the workload cluster (default `$NAMESPACE`)                                                                                                                                       |
 | POD_NETWORK_CIDR                           | Range of IP addresses for the pod network (default `192.168.0.0/16`)                                                  |
 | SERVICE_CIDR                               | Range of IP address for service VIPs (default `10.96.0.0/12`)                                                         |
 | VIRTINK_CONTROL_PLANE_SERVICE_TYPE         | The type of control plane service (default `NodePort`)                                                                |

diff --git a/api/v1beta1/virtinkmachine_types.go b/api/v1beta1/virtinkmachine_types.go
@@ -18,9 +18,15 @@ type VirtinkMachineSpec struct {
 
 	ProviderID *string `json:"providerID,omitempty"`
 
-	VMSpec          virtv1alpha1.VirtualMachineSpec   `json:"vmSpec"`
-	VolumeTemplates []VolumeTemplateSource            `json:"volumeTemplates,omitempty"`
-	IPPoolRef       *corev1.TypedLocalObjectReference `json:"ipPoolRef,omitempty"`
+	VirtualMachineTemplate VirtualMachineTemplateSpec        `json:"virtualMachineTemplate"`
+	VolumeTemplates        []VolumeTemplateSource            `json:"volumeTemplates,omitempty"`
+	IPPoolRef              *corev1.TypedLocalObjectReference `json:"ipPoolRef,omitempty"`
+}
+
+type VirtualMachineTemplateSpec struct {
+	// +kubebuilder:pruning:PreserveUnknownFields
+	ObjectMeta metav1.ObjectMeta               `json:"metadata,omitempty"`
+	Spec       virtv1alpha1.VirtualMachineSpec `json:"spec"`
 }
 
 type VolumeTemplateSource struct {

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkmachines.yaml
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkmachinetemplates.yaml
diff --git a/controllers/virtinkcluster_controller.go b/controllers/virtinkcluster_controller.go
@@ -82,12 +82,16 @@ func (r *VirtinkClusterReconciler) reconcile(ctx context.Context, cluster *infra
 		}
 	}
 
+	infraNamespace := cluster.Namespace
+	if cluster.Spec.ControlPlaneServiceTemplate.ObjectMeta.Namespace != "" {
+		infraNamespace = cluster.Spec.ControlPlaneServiceTemplate.ObjectMeta.Namespace
+	}
 	if !cluster.DeletionTimestamp.IsZero() {
 		if controllerutil.ContainsFinalizer(cluster, finalizer) {
 			var controlPlaneService corev1.Service
 			controlPlaneServiceKey := types.NamespacedName{
 				Name:      cluster.Name,
-				Namespace: cluster.Namespace,
+				Namespace: infraNamespace,
 			}
 			controlPlaneServiceNotFound := false
 			if err := infraClusterClient.Get(ctx, controlPlaneServiceKey, &controlPlaneService); err != nil {
@@ -124,7 +128,7 @@ func (r *VirtinkClusterReconciler) reconcile(ctx context.Context, cluster *infra
 		var controlPlaneService corev1.Service
 		controlPlaneServiceKey := types.NamespacedName{
 			Name:      cluster.Name,
-			Namespace: cluster.Namespace,
+			Namespace: infraNamespace,
 		}
 		controlPlaneServiceNotFound := false
 		if err := infraClusterClient.Get(ctx, controlPlaneServiceKey, &controlPlaneService); err != nil {

diff --git a/controllers/virtinkmachine_controller.go b/controllers/virtinkmachine_controller.go
@@ -134,12 +134,17 @@ func (r *VirtinkMachineReconciler) reconcile(ctx context.Context, machine *infra
 		}
 	}
 
+	infraNamespace := machine.Namespace
+	if machine.Spec.VirtualMachineTemplate.ObjectMeta.Namespace != "" {
+		infraNamespace = machine.Spec.VirtualMachineTemplate.ObjectMeta.Namespace
+	}
+
 	if !machine.DeletionTimestamp.IsZero() {
 		if controllerutil.ContainsFinalizer(machine, finalizer) {
 			var vm virtv1alpha1.VirtualMachine
 			vmKey := types.NamespacedName{
 				Name:      machine.Name,
-				Namespace: machine.Namespace,
+				Namespace: infraNamespace,
 			}
 			vmNotFound := false
 			if err := infraClusterClient.Get(ctx, vmKey, &vm); err != nil {
@@ -240,7 +245,7 @@ func (r *VirtinkMachineReconciler) reconcile(ctx context.Context, machine *infra
 		var vm virtv1alpha1.VirtualMachine
 		vmKey := types.NamespacedName{
 			Name:      machine.Name,
-			Namespace: machine.Namespace,
+			Namespace: infraNamespace,
 		}
 		vmNotFound := false
 		if err := infraClusterClient.Get(ctx, vmKey, &vm); err != nil {
@@ -371,7 +376,7 @@ func (r *VirtinkMachineReconciler) buildVM(ctx context.Context, machine *infrast
 			Labels:      machine.Labels,
 			Annotations: machine.Annotations,
 		},
-		Spec: machine.Spec.VMSpec,
+		Spec: machine.Spec.VirtualMachineTemplate.Spec,
 	}
 
 	for i := range vm.Spec.Volumes {
@@ -405,13 +410,18 @@ func (r *VirtinkMachineReconciler) buildVM(ctx context.Context, machine *infrast
 }
 
 func (r *VirtinkMachineReconciler) buildDataVolumes(ctx context.Context, machine *infrastructurev1beta1.VirtinkMachine) []*cdiv1beta1.DataVolume {
+	infraNamespace := machine.Namespace
+	if machine.Spec.VirtualMachineTemplate.ObjectMeta.Namespace != "" {
+		infraNamespace = machine.Spec.VirtualMachineTemplate.ObjectMeta.Namespace
+	}
+
 	dataVolumes := []*cdiv1beta1.DataVolume{}
 	for _, volume := range machine.Spec.VolumeTemplates {
 		switch {
 		case volume.DataVolume != nil:
 			dataVolume := cdiv1beta1.DataVolume{
 				ObjectMeta: metav1.ObjectMeta{
-					Namespace: machine.Namespace,
+					Namespace: infraNamespace,
 					Name:      fmt.Sprintf("%s-%s", machine.Name, volume.DataVolume.Name),
 				},
 				Spec: *volume.DataVolume.Spec.DeepCopy(),

diff --git a/controllers/virtinkmachine_controller_test.go b/controllers/virtinkmachine_controller_test.go
@@ -13,6 +13,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	capiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	infrastructurev1beta1 "github.com/smartxworks/cluster-api-provider-virtink/api/v1beta1"
@@ -23,6 +24,7 @@ var _ = Describe("VirtinkMachine controller", func() {
 		var clusterKey types.NamespacedName
 		var machineKey types.NamespacedName
 		var virtinkMachineKey types.NamespacedName
+		var virtualMachineKey types.NamespacedName
 		BeforeEach(func() {
 			By("creating a new VirtinkCluster")
 			clusterKey = types.NamespacedName{
@@ -74,6 +76,19 @@ var _ = Describe("VirtinkMachine controller", func() {
 				Namespace: "default",
 				Name:      "virtink-machine-" + uuid.New().String(),
 			}
+
+			infraNamespace := &corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "infra-namespace",
+				},
+			}
+			Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, infraNamespace))).To(Succeed())
+
+			virtualMachineKey = types.NamespacedName{
+				Namespace: infraNamespace.Name,
+				Name:      virtinkMachineKey.Name,
+			}
+
 			virtinkMachine := infrastructurev1beta1.VirtinkMachine{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      virtinkMachineKey.Name,
@@ -83,11 +98,16 @@ var _ = Describe("VirtinkMachine controller", func() {
 					},
 				},
 				Spec: infrastructurev1beta1.VirtinkMachineSpec{
-					VMSpec: virtv1alpha1.VirtualMachineSpec{
-						Instance: virtv1alpha1.Instance{
-							CPU: virtv1alpha1.CPU{
-								Sockets:        uint32(1),
-								CoresPerSocket: uint32(2),
+					VirtualMachineTemplate: infrastructurev1beta1.VirtualMachineTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Namespace: virtualMachineKey.Namespace,
+						},
+						Spec: virtv1alpha1.VirtualMachineSpec{
+							Instance: virtv1alpha1.Instance{
+								CPU: virtv1alpha1.CPU{
+									Sockets:        uint32(1),
+									CoresPerSocket: uint32(2),
+								},
 							},
 						},
 					},
@@ -109,7 +129,7 @@ var _ = Describe("VirtinkMachine controller", func() {
 			It("should not create VM", func() {
 				var vm virtv1alpha1.VirtualMachine
 				Consistently(func() bool {
-					return apierrors.IsNotFound(k8sClient.Get(ctx, virtinkMachineKey, &vm))
+					return apierrors.IsNotFound(k8sClient.Get(ctx, virtualMachineKey, &vm))
 				}).Should(BeTrue())
 			})
 		})
@@ -128,7 +148,7 @@ var _ = Describe("VirtinkMachine controller", func() {
 				It("should not create VM", func() {
 					var vm virtv1alpha1.VirtualMachine
 					Consistently(func() bool {
-						return apierrors.IsNotFound(k8sClient.Get(ctx, virtinkMachineKey, &vm))
+						return apierrors.IsNotFound(k8sClient.Get(ctx, virtualMachineKey, &vm))
 					}).Should(BeTrue())
 				})
 			})
@@ -156,9 +176,10 @@ var _ = Describe("VirtinkMachine controller", func() {
 				It("should create a virtink VM", func() {
 					var vm virtv1alpha1.VirtualMachine
 					Eventually(func() error {
-						return k8sClient.Get(ctx, virtinkMachineKey, &vm)
+						return k8sClient.Get(ctx, virtualMachineKey, &vm)
 					}, "10s").Should(Succeed())
 					Expect(vm.Spec.Volumes[0].CloudInit.UserDataBase64).To(Equal(base64.StdEncoding.EncodeToString([]byte("#cloud-init"))))
+					Expect(vm.Namespace).To(Equal("infra-namespace"))
 
 					var virtinkMachine infrastructurev1beta1.VirtinkMachine
 					Eventually(func() bool {
@@ -172,7 +193,7 @@ var _ = Describe("VirtinkMachine controller", func() {
 					It("should delete virtink VM and remove finalizer", func() {
 						var vm virtv1alpha1.VirtualMachine
 						Eventually(func() error {
-							return k8sClient.Get(ctx, virtinkMachineKey, &vm)
+							return k8sClient.Get(ctx, virtualMachineKey, &vm)
 						}, "10s").Should(Succeed())
 
 						virtinkMachine := infrastructurev1beta1.VirtinkMachine{
@@ -184,7 +205,7 @@ var _ = Describe("VirtinkMachine controller", func() {
 						Expect(k8sClient.Delete(ctx, &virtinkMachine)).To(Succeed())
 
 						Eventually(func() bool {
-							return apierrors.IsNotFound(k8sClient.Get(ctx, virtinkMachineKey, &vm))
+							return apierrors.IsNotFound(k8sClient.Get(ctx, virtualMachineKey, &vm))
 						}).Should(BeTrue())
 					})
 				})