Merge pull request kosmos-io#478 from OrangeBao/feature_coredns

feat: add core-dns

.gitignore

@@ -30,4 +30,4 @@ kube-config
 
 __debug_bin*
 
-ignore_dir
+ignore_dir

cmd/kubenest/operator/kubelet_node_helper.sh

@@ -101,8 +101,8 @@ function check() {
         if [ $? -ne 0 ]; then
             exit 1
         fi
-        echo "check(2/2): copy  kubeadm-flags.env  to create $PATH_FILE_TMP"
-        echo "y" | cp "$PATH_KUBELET_LIB/kubeadm-flags.env" "$PATH_FILE_TMP/"
+        echo "check(2/2): copy  kubeadm-flags.env  to create $PATH_FILE_TMP and remove args[cloud-provider] "
+        sed -e "s| --cloud-provider=external | |g" -e "w ${PATH_FILE_TMP}/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env"
     fi
     echo "environments is ok"
 }

deploy/crds/kosmos.io_virtualclusters.yaml

@@ -19,6 +19,9 @@ spec:
     - jsonPath: .status.phase
       name: STATUS
       type: string
+    - jsonPath: .status.updateTime
+      name: UPDATE-TIME
+      type: string
     name: v1alpha1
     schema:
       openAPIV3Schema:
@@ -139,7 +142,7 @@ spec:
                 type: string
               reason:
                 type: string
-              timeStamp:
+              updateTime:
                 format: date-time
                 type: string
             type: object

deploy/virtual-cluster-operator.yml

@@ -48,19 +48,31 @@ data:
     #!/usr/bin/env bash
 
     SCRIPT_VERSION=0.0.1
-    PATH_KUBEADM=/apps/bin/kubeadm
     # save tmp file
     PATH_FILE_TMP=/apps/conf/kosmos/tmp
-    PATH_KUBERNETES=/apps/conf/kubernetes
+    ###################################################
+    # path for kubeadm
+    PATH_KUBEADM=/usr/bin/kubeadm
+    ##################################################
+    # path for kubernetes
+    PATH_KUBERNETES=/etc/kubernetes/
     PATH_KUBERNETES_PKI="$PATH_KUBERNETES/pki"
+    # scpKCCmd.name
+    KUBELET_KUBE_CONFIG_NAME=kubelet.conf
+    ##################################################
+    # path for kubelet
+    PATH_KUBELET_LIB=/var/lib/kubelet
+    # scpKubeletConfigCmd.name
+    KUBELET_CONFIG_NAME=config.yaml
+
     # args
     DNS_ADDRESS=${2:-10.237.0.10}
     LOG_NAME=${2:-kubelet}
 
     function unjoin() {
         # before unjoin, you need delete node by kubectl
         echo "exec(1/1): kubeadm reset...."
-        echo "y" | . ${PATH_KUBEADM}
+        echo "y" | ${PATH_KUBEADM} reset
         if [ $? -ne 0 ]; then
             exit 1
         fi
@@ -69,32 +81,37 @@ data:
 
     # before join, you need upload ca.crt and kubeconfig to tmp dir!!!
     function join() {
-        echo "exec(1/6): stop containerd...."
+        echo "exec(1/7): stop containerd...."
         systemctl stop containerd
         if [ $? -ne 0 ]; then
             exit 1
         fi
-        echo "exec(2/6): copy ca.crt...."
+        echo "exec(2/7): copy ca.crt...."
         cp "$PATH_FILE_TMP/ca.crt" "$PATH_KUBERNETES_PKI/ca.crt"
         if [ $? -ne 0 ]; then
             exit 1
         fi
-        echo "exec(3/6): copy kubeconfig...."
-        cp "$PATH_FILE_TMP/kubeconfig" "$PATH_KUBERNETES/kubeconfig"
+        echo "exec(3/7): copy kubeconfig...."
+        cp "$PATH_FILE_TMP/$KUBELET_KUBE_CONFIG_NAME" "$PATH_KUBERNETES/$KUBELET_KUBE_CONFIG_NAME"
+        if [ $? -ne 0 ]; then
+            exit 1
+        fi
+        echo "exec(4/7): set core dns address...."
+        sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBELET_LIB}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME"
         if [ $? -ne 0 ]; then
             exit 1
         fi
-        echo "exec(4/6): set core dns address...."
-        sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBERNETES}/kubelet-config.yaml" "$PATH_FILE_TMP"/kubelet-config.yaml
+        echo "exec(5/7): copy kubeadm-flags.env...."
+        cp "$PATH_FILE_TMP/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env"
         if [ $? -ne 0 ]; then
             exit 1
         fi
-        echo "exec(5/6): start containerd"
+        echo "exec(6/7): start containerd"
         systemctl start containerd
         if [ $? -ne 0 ]; then
             exit 1
         fi
-        echo "exec(6/6): start kubelet...."
+        echo "exec(7/7): start kubelet...."
         systemctl start kubelet
         if [ $? -ne 0 ]; then
             exit 1
@@ -126,11 +143,13 @@ data:
     # check the environments
     function check() {
         if [ ! -d "$PATH_FILE_TMP" ]; then
-            echo "try to create $PATH_FILE_TMP"
+            echo "check(1/2): try to create $PATH_FILE_TMP"
             mkdir -p "$PATH_FILE_TMP"
             if [ $? -ne 0 ]; then
                 exit 1
             fi
+            echo "check(2/2): copy  kubeadm-flags.env  to create $PATH_FILE_TMP and remove args[cloud-provider] "
+            sed -e "s| --cloud-provider=external | |g" -e "w ${PATH_FILE_TMP}/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env"
         fi
         echo "environments is ok"
     }
@@ -217,8 +236,6 @@ data:
     streamingConnectionIdleTimeout: 0s
     syncFrequency: 0s
     volumeStatsAggPeriod: 0s
-
-
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -240,11 +257,17 @@ spec:
       serviceAccountName: virtual-cluster-operator
       containers:
       - name: virtual-cluster-operator
-        image: cis-hub-huabei-3.cmecloud.cn/vc-develop/virtual-cluster-operator:0.0.1
+        image: ghcr.io/kosmos-io/virtual-cluster-operator:__VERSION__
         imagePullPolicy: IfNotPresent
         env:
           - name: IMAGE_REPOSITIRY
-            value: repo1-cn-beijing.cr.volces.com/virtualcluster/virtualcluter
+            value: ghcr.io/kosmos-io
+          - name: IMAGE_VERSION
+            value: v1.25.7
+          - name: EXECTOR_HOST_MASTER_NODE_IP
+            value: 192.168.0.1
+          - name: EXECTOR_SHELL_PATH
+            value: /bin
         volumeMounts:
         - name: credentials
           mountPath: /etc/virtual-cluster-operator
@@ -255,6 +278,8 @@ spec:
         - name: shellscript
           mountPath: /bin/config.yaml
           subPath: config.yaml
+        - mountPath: /kosmos/manifest
+          name: components-manifest
         command:
         - virtual-cluster-operator
         - --kubeconfig=/etc/virtual-cluster-operator/kubeconfig
@@ -266,3 +291,7 @@ spec:
       - name: shellscript
         configMap:
           name: virtual-cluster-operator
+      - hostPath:
+          path: /home/vc-operator/manifest
+          type: DirectoryOrCreate
+        name: components-manifest

pkg/apis/kosmos/v1alpha1/virtualcluster_types.go

@@ -25,6 +25,7 @@ const (
 // +kubebuilder:resource:scope=Namespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:printcolumn:name="STATUS",type=string,JSONPath=`.status.phase`
+// +kubebuilder:printcolumn:name="UPDATE-TIME",type=string,JSONPath=`.status.updateTime`
 
 type VirtualCluster struct {
 	metav1.TypeMeta   `json:",inline"`
@@ -91,7 +92,7 @@ type VirtualClusterStatus struct {
 	// +optional
 	Reason string `json:"reason,omitempty" protobuf:"bytes,4,opt,name=reason"`
 	// +optional
-	TimeStamp *metav1.Time `json:"timeStamp,omitempty" protobuf:"bytes,7,opt,name=timeStamp"`
+	UpdateTime *metav1.Time `json:"updateTime,omitempty" protobuf:"bytes,7,opt,name=updateTime"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/kubenest/constants/constant.go

@@ -87,6 +87,12 @@ const (
 	NodeFreeState               = "free"
 
 	WaitAllPodsRunningTimeoutSeconds = 1800
+
+	// core-dns
+	KubeDNSSVCName = "kube-dns"
+	// nolint
+	HostCoreDnsComponents    = "host-core-dns-components"
+	VirtualCoreDnsComponents = "virtual-core-dns-components"
 )
 
 type Action string

pkg/kubenest/controller/virtualcluster.node.controller/join-worker.go

@@ -29,6 +29,15 @@ func getJoinCmdStr(log string) (string, error) {
 	return fmt.Sprintf("kubeadm join %s", strs[1]), nil
 }
 
+func isNodeReady(conditions []v1.NodeCondition) bool {
+	for _, condition := range conditions {
+		if condition.Type == v1.NodeReady && condition.Status == v1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}
+
 func (r *NodeController) WaitNodeReady(ctx context.Context, nodeInfo vcrnodepoolcontroller.NodeItem, k8sClient kubernetes.Interface) error {
 	waitCtx, cancel := context.WithTimeout(ctx, 60*time.Second) // total waiting time
 	defer cancel()
@@ -38,7 +47,7 @@ func (r *NodeController) WaitNodeReady(ctx context.Context, nodeInfo vcrnodepool
 	wait.UntilWithContext(waitCtx, func(ctx context.Context) {
 		node, err := k8sClient.CoreV1().Nodes().Get(waitCtx, nodeInfo.Name, metav1.GetOptions{})
 		if err == nil {
-			if node.Status.Phase == v1.NodeRunning {
+			if isNodeReady(node.Status.Conditions) {
 				klog.V(4).Infof("node %s is ready", nodeInfo.Name)
 				isReady = true
 				cancel()

pkg/kubenest/controller/virtualcluster.node.controller/node_controller.go

@@ -11,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -38,7 +39,7 @@ func (r *NodeController) SetupWithManager(mgr manager.Manager) error {
 		r.Client = mgr.GetClient()
 	}
 
-	skipEvent := func(obj client.Object) bool {
+	skipEvent := func(_ client.Object) bool {
 		return true
 	}
 
@@ -150,15 +151,27 @@ func (r *NodeController) GetNodePool(ctx context.Context) (map[string]vcrnodepoo
 }
 
 func (r *NodeController) UpdateVirtualClusterStatus(ctx context.Context, virtualCluster v1alpha1.VirtualCluster, status v1alpha1.Phase, reason string) error {
-	updateVirtualCluster := virtualCluster.DeepCopy()
-	updateVirtualCluster.Status.Phase = status
-	updateVirtualCluster.Status.Reason = reason
-	timestamp := metav1.Now()
-	updateVirtualCluster.Status.TimeStamp = &timestamp
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		targetObj := v1alpha1.VirtualCluster{}
+		if err := r.Get(ctx, types.NamespacedName{Name: virtualCluster.Name, Namespace: virtualCluster.Namespace}, &targetObj); err != nil {
+			return err
+		}
+		updateVirtualCluster := targetObj.DeepCopy()
+		updateVirtualCluster.Status.Phase = status
+		updateVirtualCluster.Status.Reason = reason
+		updateTime := metav1.Now()
+		updateVirtualCluster.Status.UpdateTime = &updateTime
+
+		if err := r.Update(ctx, updateVirtualCluster); err != nil {
+			return err
+		}
+		return nil
+	})
 
-	if err := r.Update(ctx, updateVirtualCluster); err != nil {
-		return fmt.Errorf("update virtualcluster %s status failed: %s", virtualCluster.Name, err)
+	if retryErr != nil {
+		return fmt.Errorf("update virtualcluster %s status failed: %s", virtualCluster.Name, retryErr)
 	}
+
 	return nil
 }
 
@@ -235,7 +248,7 @@ func (r *NodeController) Reconcile(ctx context.Context, request reconcile.Reques
 		return reconcile.Result{}, nil
 	}
 
-	if virtualCluster.Status.Phase != v1alpha1.Initialized {
+	if virtualCluster.Status.Phase == v1alpha1.Preparing {
 		klog.V(4).Infof("virtualcluster wait cluster ready, cluster name: %s", virtualCluster.Name)
 		return reconcile.Result{RequeueAfter: utils.DefaultRequeueTime}, nil
 	}

pkg/kubenest/controller/virtualcluster.node.controller/share.go

@@ -2,7 +2,6 @@ package vcnodecontroller
 
 import (
 	"context"
-	"fmt"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -16,7 +15,7 @@ func (r *NodeController) UpdateNodePoolState(ctx context.Context, nodeName strin
 	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
 		nodePool := v1.ConfigMap{}
 		if err := r.Client.Get(ctx, types.NamespacedName{Name: NodePoolCMName, Namespace: NodePoolCMNS}, &nodePool); err != nil {
-			return fmt.Errorf("get node-pool failed: %v", err)
+			return err
 		}
 
 		updateNodePool := nodePool.DeepCopy()

pkg/kubenest/controller/virtualcluster.node.controller/unjoin-worker.go

@@ -69,6 +69,8 @@ func (r *NodeController) unjoinNode(ctx context.Context, nodeInfos []vcrnodepool
 		}
 		klog.V(4).Infof("remove node from cluster successed, node name: %s", nodeInfo.Name)
 
+		// TODO: reset kubeadm-flags.env
+
 		// TODO： move to node pool controller，  add node to host cluster
 		if err := r.joinNodeToHost(ctx, nodeInfo); err != nil {
 			klog.Errorf("join node %s to host cluster failed: %s", nodeInfo.Name, err)

pkg/kubenest/init.go

@@ -6,6 +6,7 @@ import (
 
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 
@@ -30,6 +31,7 @@ type initData struct {
 	clusterIps            []string
 	remoteClient          clientset.Interface
 	kosmosClient          versioned.Interface
+	dynamicClient         *dynamic.DynamicClient
 	virtualClusterDataDir string
 	privateRegistry       string
 	externalIP            string
@@ -57,7 +59,9 @@ func NewInitPhase(opts *InitOptions, hostPortManager *vcnodecontroller.HostPortM
 	initPhase.AppendTask(tasks.NewCheckApiserverHealthTask())
 	initPhase.AppendTask(tasks.NewComponentTask())
 	initPhase.AppendTask(tasks.NewCheckControlPlaneTask())
-	initPhase.AppendTask(tasks.NewComponentsFromManifestsTask())
+	// create core-dns
+	initPhase.AppendTask(tasks.NewCoreDNSTask())
+	initPhase.AppendTask(tasks.NewComponentsFromManifestsTask()) // add server
 
 	initPhase.SetDataInitializer(func() (workflow.RunData, error) {
 		return newRunData(opts, hostPortManager)
@@ -126,6 +130,11 @@ func newRunData(opt *InitOptions, hostPortManager *vcnodecontroller.HostPortMana
 	}
 	var remoteClient clientset.Interface = localClusterClient
 
+	dynamicClient, err := dynamic.NewForConfig(opt.Kubeconfig)
+	if err != nil {
+		return nil, err
+	}
+
 	kosmosClient, err := versioned.NewForConfig(opt.Kubeconfig)
 	if err != nil {
 		return nil, fmt.Errorf("error when creating  kosmosClient client, err: %w", err)
@@ -154,6 +163,7 @@ func newRunData(opt *InitOptions, hostPortManager *vcnodecontroller.HostPortMana
 		controlplaneAddr:      address,
 		clusterIps:            clusterIps,
 		remoteClient:          remoteClient,
+		dynamicClient:         dynamicClient,
 		kosmosClient:          kosmosClient,
 		virtualClusterDataDir: opt.virtualClusterDataDir,
 		privateRegistry:       utils.DefaultImageRepository,
@@ -222,3 +232,7 @@ func (i initData) VirtualClusterVersion() string {
 func (i initData) ExternalIP() string {
 	return i.externalIP
 }
+
+func (i initData) DynamicClient() *dynamic.DynamicClient {
+	return i.dynamicClient
+}