Support templated deployment for cloud provider

internal/controller/huaweicloudmachine_controller.go

@@ -125,6 +125,7 @@ func (r *HuaweiCloudMachineReconciler) Reconcile(ctx context.Context, req ctrl.R
 		Machine:      machine,
 		InfraCluster: infraCluster,
 		HCMachine:    hcMachine,
+		Credentials:  r.Credentials,
 	})
 	if err != nil {
 		log.Error(err, "failed to create scope")
@@ -223,7 +224,6 @@ func (r *HuaweiCloudMachineReconciler) SetupWithManager(mgr ctrl.Manager) error
 
 func (r *HuaweiCloudMachineReconciler) reconcileDelete(machineScope *scope.MachineScope, _ hwbasic.ClusterScoper, ecsScope scope.ECSScope) (ctrl.Result, error) {
 	machineScope.Logger.Info("Handling deleted HuaweiCloudMachine")
-
 	ecsSvc, err := ecs.NewService(ecsScope)
 	if err != nil {
 		machineScope.Logger.Error(err, "failed to get ECS service")

pkg/scope/machine.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
@@ -28,6 +29,7 @@ type MachineScopeParams struct {
 	Machine      *clusterv1.Machine
 	InfraCluster ECSScope
 	HCMachine    *infrav1.HuaweiCloudMachine
+	Credentials  *basic.Credentials
 }
 
 // NewMachineScope creates a new MachineScope from the supplied parameters.
@@ -66,6 +68,7 @@ func NewMachineScope(params MachineScopeParams) (*MachineScope, error) {
 		Machine:      params.Machine,
 		InfraCluster: params.InfraCluster,
 		HCMachine:    params.HCMachine,
+		Credentials:  params.Credentials,
 	}, nil
 }
 
@@ -79,6 +82,7 @@ type MachineScope struct {
 	Machine      *clusterv1.Machine
 	InfraCluster ECSScope
 	HCMachine    *infrav1.HuaweiCloudMachine
+	Credentials  *basic.Credentials
 }
 
 // Name returns the HuaweiCloudMachine name.
@@ -203,6 +207,10 @@ func (m *MachineScope) GetRawBootstrapDataWithFormat() ([]byte, string, error) {
 	return value, string(secret.Data["format"]), nil
 }
 
+func (m *MachineScope) GetCredentials() *basic.Credentials {
+	return m.Credentials
+}
+
 // PatchObject persists the machine spec and status.
 func (m *MachineScope) PatchObject() error {
 	// Always update the readyCondition by summarizing the state of other conditions.

pkg/services/ecs/cloudconfig.go

@@ -0,0 +1,89 @@
+package ecs
+
+import (
+	"bytes"
+	"text/template"
+
+	"gopkg.in/yaml.v2"
+)
+
+// CloudConfig holds the configuration for the cloud provider.
+type CloudConfig struct {
+	Region    string
+	AccessKey string
+	SecretKey string
+	VPCID     string
+	SubnetID  string
+}
+
+// CloudInitConfig represents the structure of a cloud-init configuration.
+type CloudInitConfig struct {
+	WriteFiles []*WriteFile `yaml:"write_files,omitempty"`
+	RunCmd     []string     `yaml:"runcmd,omitempty"`
+}
+
+// WriteFile represents a file to be written by cloud-init.
+type WriteFile struct {
+	Path        string `yaml:"path"`
+	Owner       string `yaml:"owner"`
+	Permissions string `yaml:"permissions"`
+	Content     string `yaml:"content"`
+}
+
+func (c *CloudConfig) genCloudProviderSecretTask() (writeFile *WriteFile, runCmd []string, err error) {
+	contentTemplate := "[Global]\n  region={{.Region}}\n  access-key={{.AccessKey}}\n  secret-key={{.SecretKey}}\n\n[Vpc]\n  id={{.VPCID}}\n  subnet-id={{.SubnetID}}\n"
+
+	var contentBuffer bytes.Buffer
+	tmpl, err := template.New("content").Parse(contentTemplate)
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := tmpl.Execute(&contentBuffer, c); err != nil {
+		return nil, nil, err
+	}
+
+	writeFile = &WriteFile{
+		Path:        "/etc/kubernetes/cloud-config",
+		Owner:       "root:root",
+		Permissions: "0644",
+		Content:     contentBuffer.String(),
+	}
+
+	runCmd = []string{
+		"if ! kubectl get secret cloud-config; then kubectl create secret generic cloud-config --from-file=/etc/kubernetes/cloud-config; fi",
+		// "rm -rf /etc/kubernetes/cloud-config",
+	}
+
+	return writeFile, runCmd, nil
+}
+
+// appendCloudConfig appends the cloud provider secret to the cloud-init configuration.
+func (c *CloudConfig) appendCloudConfig(cloudConfYaml []byte) ([]byte, error) {
+	configHeader := "## template: jinja\n#cloud-config\n"
+
+	// Parse the existing cloud-init YAML
+	var cloudInitConfig CloudInitConfig
+	if err := yaml.Unmarshal(cloudConfYaml, &cloudInitConfig); err != nil {
+		return nil, err
+	}
+
+	// task1: generate the cloud provider secret
+	writeFile, runCmd, err := c.genCloudProviderSecretTask()
+	if err != nil {
+		return nil, err
+	}
+	cloudInitConfig.WriteFiles = append(cloudInitConfig.WriteFiles, writeFile)
+	cloudInitConfig.RunCmd = append(cloudInitConfig.RunCmd, runCmd...)
+
+	// Marshal the updated cloud-init configuration back to YAML
+	updatedCloudConfYaml, err := yaml.Marshal(&cloudInitConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	// Combine configHeader and updatedCloudConfYaml
+	var finalYaml bytes.Buffer
+	finalYaml.WriteString(configHeader)
+	finalYaml.Write(updatedCloudConfYaml)
+	return finalYaml.Bytes(), nil
+}

pkg/services/ecs/instance.go

@@ -169,6 +169,22 @@ func (s *Service) CreateInstance(scope *scope.MachineScope, userData []byte,
 		input.PublicIPOnLaunch = ptr.To(false)
 	}
 
+	if scope.IsControlPlane() {
+		cloudConf := &CloudConfig{
+			Region:    s.scope.Region(),
+			AccessKey: scope.Credentials.AK,
+			SecretKey: scope.Credentials.SK,
+			VPCID:     s.scope.VPC().Id,
+			SubnetID:  subnetID,
+		}
+		userData, err = cloudConf.appendCloudConfig(userData)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to append cloud config")
+		}
+	}
+
+	klog.Infof("userData:\n%s\n", string(userData))
+
 	input.UserData = ptr.To[string](base64.StdEncoding.EncodeToString(userData))
 
 	// Set security groups.
@@ -186,7 +202,6 @@ func (s *Service) CreateInstance(scope *scope.MachineScope, userData []byte,
 	// Set the providerID and instanceID as soon as we create an instance so that we keep it in case of errors afterward
 	scope.SetProviderID(out.ID, out.AvailabilityZone)
 	scope.SetInstanceID(out.ID)
-
 	return out, nil
 }
 

templates/cluster-template.yaml

@@ -113,4 +113,108 @@ spec:
     spec:
       imageRef: "${ECS_IMAGE_ID}"
       flavorRef: "${HC_NODE_MACHINE_TYPE}"
-      publicIP: true
+      publicIP: true
+---
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: crs-ccm
+spec:
+  clusterSelector:
+    matchLabels:
+      ccm: external
+  resources:
+  - kind: ConfigMap
+    name: cloud-controller-manager-addon
+  strategy: ApplyOnce
+---
+apiVersion: v1
+data:
+  hw-cp-external.yaml: |
+    ---
+    apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      name: huawei-cloud-controller-manager
+      namespace: kube-system
+      labels:
+        k8s-app: huawei-cloud-controller-manager
+    spec:
+      selector:
+        matchLabels:
+          k8s-app: huawei-cloud-controller-manager
+      template:
+        metadata:
+          labels:
+            k8s-app: huawei-cloud-controller-manager
+        spec:
+          nodeSelector:
+            kubernetes.io/os: linux
+          securityContext:
+            runAsUser: 1001
+          tolerations:
+            - key: node.cloudprovider.kubernetes.io/uninitialized
+              value: "true"
+              effect: NoSchedule
+            - key: node-role.kubernetes.io/master
+              effect: NoSchedule
+            - key: node-role.kubernetes.io/control-plane
+              effect: NoSchedule
+          serviceAccountName: cloud-controller-manager
+          containers:
+            - name: huawei-cloud-controller-manager
+              image: swr.cn-north-4.myhuaweicloud.com/k8s-cloudprovider/huawei-cloud-controller-manager:v0.26.9
+              args:
+                - /bin/huawei-cloud-controller-manager
+                - --v=5
+                - --cloud-config=/etc/config/cloud-config
+                - --cloud-provider=huaweicloud
+                - --use-service-account-credentials=true
+                - --node-status-update-frequency=5s
+                - --node-monitor-period=5s
+                - --leader-elect-lease-duration=30s
+                - --leader-elect-renew-deadline=20s
+                - --leader-elect-retry-period=2s
+              volumeMounts:
+                - mountPath: /etc/kubernetes
+                  name: k8s-certs
+                  readOnly: true
+                - mountPath: /etc/ssl/certs
+                  name: ca-certs
+                  readOnly: true
+                - mountPath: /etc/config
+                  name: cloud-config-volume
+                  readOnly: true
+                - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+                  name: flexvolume-dir
+              resources:
+                requests:
+                  cpu: 200m
+                  memory: 100Mi
+                limits:
+                  cpu: 2
+                  memory: 2Gi
+          hostNetwork: true
+          volumes:
+            - hostPath:
+                path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+                type: DirectoryOrCreate
+              name: flexvolume-dir
+            - hostPath:
+                path: /etc/kubernetes
+                type: DirectoryOrCreate
+              name: k8s-certs
+            - hostPath:
+                path: /etc/ssl/certs
+                type: DirectoryOrCreate
+              name: ca-certs
+            - name: cloud-config-volume
+              secret:
+                secretName: cloud-config
+kind: ConfigMap
+metadata:
+  annotations:
+    note: generated
+  labels:
+    type: generated
+  name: cloud-controller-manager-addon