Skip to content

Commit

Permalink
Support templated deployment for cloud provider
Browse files Browse the repository at this point in the history
Signed-off-by: bo.jiang <bo.jiang@daocloud.io>
  • Loading branch information
ErikJiang committed Feb 21, 2025
1 parent 0933be9 commit 9b85788
Show file tree
Hide file tree
Showing 5 changed files with 220 additions and 3 deletions.
2 changes: 1 addition & 1 deletion internal/controller/huaweicloudmachine_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ func (r *HuaweiCloudMachineReconciler) Reconcile(ctx context.Context, req ctrl.R
Machine: machine,
InfraCluster: infraCluster,
HCMachine: hcMachine,
Credentials: r.Credentials,
})
if err != nil {
log.Error(err, "failed to create scope")
Expand Down Expand Up @@ -223,7 +224,6 @@ func (r *HuaweiCloudMachineReconciler) SetupWithManager(mgr ctrl.Manager) error

func (r *HuaweiCloudMachineReconciler) reconcileDelete(machineScope *scope.MachineScope, _ hwbasic.ClusterScoper, ecsScope scope.ECSScope) (ctrl.Result, error) {
machineScope.Logger.Info("Handling deleted HuaweiCloudMachine")

ecsSvc, err := ecs.NewService(ecsScope)
if err != nil {
machineScope.Logger.Error(err, "failed to get ECS service")
Expand Down
8 changes: 8 additions & 0 deletions pkg/scope/machine.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/base64"

"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
Expand All @@ -28,6 +29,7 @@ type MachineScopeParams struct {
Machine *clusterv1.Machine
InfraCluster ECSScope
HCMachine *infrav1.HuaweiCloudMachine
Credentials *basic.Credentials
}

// NewMachineScope creates a new MachineScope from the supplied parameters.
Expand Down Expand Up @@ -66,6 +68,7 @@ func NewMachineScope(params MachineScopeParams) (*MachineScope, error) {
Machine: params.Machine,
InfraCluster: params.InfraCluster,
HCMachine: params.HCMachine,
Credentials: params.Credentials,
}, nil
}

Expand All @@ -79,6 +82,7 @@ type MachineScope struct {
Machine *clusterv1.Machine
InfraCluster ECSScope
HCMachine *infrav1.HuaweiCloudMachine
Credentials *basic.Credentials
}

// Name returns the HuaweiCloudMachine name.
Expand Down Expand Up @@ -203,6 +207,10 @@ func (m *MachineScope) GetRawBootstrapDataWithFormat() ([]byte, string, error) {
return value, string(secret.Data["format"]), nil
}

func (m *MachineScope) GetCredentials() *basic.Credentials {
return m.Credentials
}

// PatchObject persists the machine spec and status.
func (m *MachineScope) PatchObject() error {
// Always update the readyCondition by summarizing the state of other conditions.
Expand Down
89 changes: 89 additions & 0 deletions pkg/services/ecs/cloudconfig.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package ecs

import (
"bytes"
"text/template"

"gopkg.in/yaml.v2"
)

// CloudConfig holds the configuration for the cloud provider.
type CloudConfig struct {
Region string
AccessKey string
SecretKey string
VPCID string
SubnetID string
}

// CloudInitConfig represents the structure of a cloud-init configuration.
type CloudInitConfig struct {
WriteFiles []*WriteFile `yaml:"write_files,omitempty"`
RunCmd []string `yaml:"runcmd,omitempty"`
}

// WriteFile represents a file to be written by cloud-init.
type WriteFile struct {
Path string `yaml:"path"`
Owner string `yaml:"owner"`
Permissions string `yaml:"permissions"`
Content string `yaml:"content"`
}

func (c *CloudConfig) genCloudProviderSecretTask() (writeFile *WriteFile, runCmd []string, err error) {
contentTemplate := "[Global]\n region={{.Region}}\n access-key={{.AccessKey}}\n secret-key={{.SecretKey}}\n\n[Vpc]\n id={{.VPCID}}\n subnet-id={{.SubnetID}}\n"

var contentBuffer bytes.Buffer
tmpl, err := template.New("content").Parse(contentTemplate)
if err != nil {
return nil, nil, err
}
if err := tmpl.Execute(&contentBuffer, c); err != nil {
return nil, nil, err
}

writeFile = &WriteFile{
Path: "/etc/kubernetes/cloud-config",
Owner: "root:root",
Permissions: "0644",
Content: contentBuffer.String(),
}

runCmd = []string{
"if ! kubectl get secret cloud-config; then kubectl create secret generic cloud-config --from-file=/etc/kubernetes/cloud-config; fi",
// "rm -rf /etc/kubernetes/cloud-config",
}

return writeFile, runCmd, nil
}

// appendCloudConfig appends the cloud provider secret to the cloud-init configuration.
func (c *CloudConfig) appendCloudConfig(cloudConfYaml []byte) ([]byte, error) {
configHeader := "## template: jinja\n#cloud-config\n"

// Parse the existing cloud-init YAML
var cloudInitConfig CloudInitConfig
if err := yaml.Unmarshal(cloudConfYaml, &cloudInitConfig); err != nil {
return nil, err
}

// task1: generate the cloud provider secret
writeFile, runCmd, err := c.genCloudProviderSecretTask()
if err != nil {
return nil, err
}
cloudInitConfig.WriteFiles = append(cloudInitConfig.WriteFiles, writeFile)
cloudInitConfig.RunCmd = append(cloudInitConfig.RunCmd, runCmd...)

// Marshal the updated cloud-init configuration back to YAML
updatedCloudConfYaml, err := yaml.Marshal(&cloudInitConfig)
if err != nil {
return nil, err
}

// Combine configHeader and updatedCloudConfYaml
var finalYaml bytes.Buffer
finalYaml.WriteString(configHeader)
finalYaml.Write(updatedCloudConfYaml)
return finalYaml.Bytes(), nil
}
18 changes: 17 additions & 1 deletion pkg/services/ecs/instance.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,22 @@ func (s *Service) CreateInstance(scope *scope.MachineScope, userData []byte,
input.PublicIPOnLaunch = ptr.To(false)
}

if scope.IsControlPlane() {
cloudConf := &CloudConfig{
Region: s.scope.Region(),
AccessKey: scope.Credentials.AK,
SecretKey: scope.Credentials.SK,
VPCID: s.scope.VPC().Id,
SubnetID: subnetID,
}
userData, err = cloudConf.appendCloudConfig(userData)
if err != nil {
return nil, errors.Wrap(err, "failed to append cloud config")
}
}

klog.Infof("userData:\n%s\n", string(userData))

input.UserData = ptr.To[string](base64.StdEncoding.EncodeToString(userData))

// Set security groups.
Expand All @@ -186,7 +202,7 @@ func (s *Service) CreateInstance(scope *scope.MachineScope, userData []byte,
// Set the providerID and instanceID as soon as we create an instance so that we keep it in case of errors afterward
scope.SetProviderID(out.ID, out.AvailabilityZone)
scope.SetInstanceID(out.ID)

// scope.GetCredentials().AK
return out, nil
}

Expand Down
106 changes: 105 additions & 1 deletion templates/cluster-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -113,4 +113,108 @@ spec:
spec:
imageRef: "${ECS_IMAGE_ID}"
flavorRef: "${HC_NODE_MACHINE_TYPE}"
publicIP: true
publicIP: true
---
apiVersion: addons.cluster.x-k8s.io/v1beta1
kind: ClusterResourceSet
metadata:
name: crs-ccm
spec:
clusterSelector:
matchLabels:
ccm: external
resources:
- kind: ConfigMap
name: cloud-controller-manager-addon
strategy: ApplyOnce
---
apiVersion: v1
data:
hw-cp-external.yaml: |
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: huawei-cloud-controller-manager
namespace: kube-system
labels:
k8s-app: huawei-cloud-controller-manager
spec:
selector:
matchLabels:
k8s-app: huawei-cloud-controller-manager
template:
metadata:
labels:
k8s-app: huawei-cloud-controller-manager
spec:
nodeSelector:
kubernetes.io/os: linux
securityContext:
runAsUser: 1001
tolerations:
- key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
effect: NoSchedule
- key: node-role.kubernetes.io/master
effect: NoSchedule
- key: node-role.kubernetes.io/control-plane
effect: NoSchedule
serviceAccountName: cloud-controller-manager
containers:
- name: huawei-cloud-controller-manager
image: swr.cn-north-4.myhuaweicloud.com/k8s-cloudprovider/huawei-cloud-controller-manager:v0.26.9
args:
- /bin/huawei-cloud-controller-manager
- --v=5
- --cloud-config=/etc/config/cloud-config
- --cloud-provider=huaweicloud
- --use-service-account-credentials=true
- --node-status-update-frequency=5s
- --node-monitor-period=5s
- --leader-elect-lease-duration=30s
- --leader-elect-renew-deadline=20s
- --leader-elect-retry-period=2s
volumeMounts:
- mountPath: /etc/kubernetes
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/config
name: cloud-config-volume
readOnly: true
- mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
name: flexvolume-dir
resources:
requests:
cpu: 200m
memory: 100Mi
limits:
cpu: 2
memory: 2Gi
hostNetwork: true
volumes:
- hostPath:
path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
type: DirectoryOrCreate
name: flexvolume-dir
- hostPath:
path: /etc/kubernetes
type: DirectoryOrCreate
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
type: DirectoryOrCreate
name: ca-certs
- name: cloud-config-volume
secret:
secretName: cloud-config
kind: ConfigMap
metadata:
annotations:
note: generated
labels:
type: generated
name: cloud-controller-manager-addon

0 comments on commit 9b85788

Please sign in to comment.