From 44e2fd80891d555915aa2f65a2faf5164b8752f7 Mon Sep 17 00:00:00 2001 From: luoyuanze Date: Wed, 25 Dec 2024 11:15:16 +0800 Subject: [PATCH] Add NodeLocalDNS configuration to the kubenest workflow. Signed-off-by: luoyuanze --- .../kosmos.io_kubenestconfigurations.yaml | 6 ++++ deploy/crds/kosmos.io_virtualclusters.yaml | 5 ++- hack/k8s-in-k8s/kubelet_node_helper.sh | 10 +++++- .../v1alpha1/kubenestconfiguration_types.go | 3 ++ pkg/generated/openapi/zz_generated.openapi.go | 6 ++++ pkg/kubenest/constants/constant.go | 6 ++++ .../workflow/task/task.go | 7 ++-- pkg/kubenest/tasks/manifests_components.go | 34 +++++++++++++++++++ 8 files changed, 73 insertions(+), 4 deletions(-) diff --git a/deploy/crds/kosmos.io_kubenestconfigurations.yaml b/deploy/crds/kosmos.io_kubenestconfigurations.yaml index d1f1c02d3..4d5387ec2 100644 --- a/deploy/crds/kosmos.io_kubenestconfigurations.yaml +++ b/deploy/crds/kosmos.io_kubenestconfigurations.yaml @@ -57,6 +57,9 @@ spec: type: string etcdUnitSize: type: string + externalPort: + format: int32 + type: integer forceDestroy: description: todo Group according to the parameters of apiserver, etcd, coredns, etc. ForceDestroy indicates whether to force destroy @@ -77,6 +80,9 @@ spec: type: string type: array type: object + useNodeLocalDNS: + default: false + type: boolean useTenantDNS: default: false type: boolean diff --git a/deploy/crds/kosmos.io_virtualclusters.yaml b/deploy/crds/kosmos.io_virtualclusters.yaml index 5e2a2c617..cb72f1aa2 100644 --- a/deploy/crds/kosmos.io_virtualclusters.yaml +++ b/deploy/crds/kosmos.io_virtualclusters.yaml @@ -75,8 +75,8 @@ spec: etcdUnitSize: type: string externalPort: - type: integer format: int32 + type: integer forceDestroy: description: todo Group according to the parameters of apiserver, etcd, coredns, etc. ForceDestroy indicates whether to force @@ -98,6 +98,9 @@ spec: type: string type: array type: object + useNodeLocalDNS: + default: false + type: boolean useTenantDNS: default: false type: boolean diff --git a/hack/k8s-in-k8s/kubelet_node_helper.sh b/hack/k8s-in-k8s/kubelet_node_helper.sh index d02e75012..c780ca762 100755 --- a/hack/k8s-in-k8s/kubelet_node_helper.sh +++ b/hack/k8s-in-k8s/kubelet_node_helper.sh @@ -8,6 +8,7 @@ LOG_NAME=${2:-kubelet} JOIN_HOST=$2 JOIN_TOKEN=$3 JOIN_CA_HASH=$4 +NODE_LOCAL_DNS_ADDRESS=$3 function unjoin() { # before unjoin, you need delete node by kubectl @@ -248,7 +249,14 @@ function join() { exit 1 fi echo "exec(4/8): set core dns address...." - sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME" + if [ -n "$NODE_LOCAL_DNS_ADDRESS" ]; then + sed -e "/__DNS_ADDRESS__/i - ${NODE_LOCAL_DNS_ADDRESS}" \ + -e "s|__DNS_ADDRESS__|${DNS_ADDRESS}|g" \ + "$PATH_FILE_TMP/$KUBELET_CONFIG_NAME" \ + > "${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}" + else + sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME" + fi if [ $? -ne 0 ]; then exit 1 fi diff --git a/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go b/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go index 5fd4cd1f5..2122b99d6 100644 --- a/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go +++ b/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go @@ -91,6 +91,9 @@ type KubeInKubeConfig struct { UseTenantDNS bool `yaml:"useTenantDNS" json:"useTenantDNS,omitempty"` // +optional ExternalPort int32 `json:"externalPort,omitempty"` + // +kubebuilder:default=false + // +optional + UseNodeLocalDNS bool `yaml:"useNodeLocalDNS" json:"useNodeLocalDNS,omitempty"` } // TenantEntrypoint contains the configuration for the tenant entrypoint. diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index a9a579096..4d2befe7f 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -1995,6 +1995,12 @@ func schema_pkg_apis_kosmos_v1alpha1_KubeInKubeConfig(ref common.ReferenceCallba Format: "int32", }, }, + "useNodeLocalDNS": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, }, }, }, diff --git a/pkg/kubenest/constants/constant.go b/pkg/kubenest/constants/constant.go index f15d9440b..3d24c5111 100644 --- a/pkg/kubenest/constants/constant.go +++ b/pkg/kubenest/constants/constant.go @@ -137,6 +137,12 @@ const ( //in virtual cluster APIServerExternalService = "api-server-external-service" + + //nodelocaldns + NodeLocalDNSComponentName = "virtual-node-local-dns" + NodeLocalDNSIp = "169.254.20.10" + NodeLocalDNSClusterDomain = "cluster.local" + NodeLocalDNSService = "__PILLAR__DNS__SERVER__" ) type Action string diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go index 118551e5d..ff549b0e1 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go @@ -254,9 +254,12 @@ func NewRemoteNodeJoinTask() Task { Retry: true, Run: func(ctx context.Context, to TaskOpt, _ interface{}) (interface{}, error) { exectHelper := exector.NewExectorHelper(to.NodeInfo.Spec.NodeIP, "") - + baseCmd := fmt.Sprintf("bash %s join %s", env.GetExectorShellName(), to.KubeDNSAddress) + if to.VirtualCluster.Spec.KubeInKubeConfig != nil && to.VirtualCluster.Spec.KubeInKubeConfig.UseNodeLocalDNS { + baseCmd = fmt.Sprintf("bash %s join %s %s", env.GetExectorShellName(), to.KubeDNSAddress, constants.NodeLocalDNSIp) + } joinCmd := &exector.CMDExector{ - Cmd: fmt.Sprintf("bash %s join %s", env.GetExectorShellName(), to.KubeDNSAddress), + Cmd: baseCmd, } to.Loger().Infof("join node %s with cmd: %s", to.NodeInfo.Name, joinCmd.Cmd) ret := exectHelper.DoExector(ctx.Done(), joinCmd) diff --git a/pkg/kubenest/tasks/manifests_components.go b/pkg/kubenest/tasks/manifests_components.go index 6b59d9866..4e8b3f7fc 100644 --- a/pkg/kubenest/tasks/manifests_components.go +++ b/pkg/kubenest/tasks/manifests_components.go @@ -2,6 +2,7 @@ package tasks import ( "context" + "fmt" "os" "path/filepath" @@ -15,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" @@ -95,6 +97,11 @@ func applyComponentsManifests(r workflow.RunData) error { templatedMapping["KUBE_PROXY_KUBECONFIG"] = string(secret.Data[constants.KubeConfig]) imageRepository, _ := util.GetImageMessage() templatedMapping["ImageRepository"] = imageRepository + + templatedMapping["PillarLocalDNS"] = constants.NodeLocalDNSIp + templatedMapping["PillarDNSDomain"] = constants.NodeLocalDNSClusterDomain + templatedMapping["PillarDNSServer"] = "" + for k, v := range data.PluginOptions() { templatedMapping[k] = v } @@ -111,6 +118,7 @@ func applyComponentsManifests(r workflow.RunData) error { } UseTenantDNS := data.VirtualCluster().Spec.KubeInKubeConfig != nil && data.VirtualCluster().Spec.KubeInKubeConfig.UseTenantDNS + UseNodeLocalDNS := data.VirtualCluster().Spec.KubeInKubeConfig != nil && data.VirtualCluster().Spec.KubeInKubeConfig.UseNodeLocalDNS skipComponents := getSkipComponentsForVirtualCluster([]*SkipComponentCondition{ { @@ -122,6 +130,11 @@ func applyComponentsManifests(r workflow.RunData) error { Condition: !keepalivedEnable, ComponentName: constants.VipKeepalivedComponentName, }, + { + // skip nodelocaldns component if nodelocaldns is not enabled + Condition: !UseNodeLocalDNS, + ComponentName: constants.NodeLocalDNSComponentName, + }, }) for _, component := range components { @@ -130,6 +143,14 @@ func applyComponentsManifests(r workflow.RunData) error { klog.V(2).Infof("Deploy component %s skipped", component.Name) continue } + if component.Name == constants.NodeLocalDNSComponentName { + kubeDNSIP, err := getKubeDNSClusterIP(config) + if err != nil { + return errors.Wrap(err, "Failed to get kube-dns ClusterIP") + } + klog.Infof("kube-dns CLUSTER-IP: %s", kubeDNSIP) + templatedMapping["PillarClusterDNS"] = kubeDNSIP + } err = applyTemplatedManifests(component.Name, dynamicClient, component.Path, templatedMapping) if err != nil { return err @@ -215,3 +236,16 @@ func applyTemplatedManifests(component string, dynamicClient dynamic.Interface, } return nil } +func getKubeDNSClusterIP(config *rest.Config) (string, error) { + client, err := clientset.NewForConfig(config) + if err != nil { + return "", fmt.Errorf("failed to create kubernetes client: %v", err) + } + + svc, err := client.CoreV1().Services("kube-system").Get(context.TODO(), "kube-dns", metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("failed to get kube-dns service: %v", err) + } + + return svc.Spec.ClusterIP, nil +}