* [FEATURE] k8s 1.20后续版本默认容器运行时采用containerd，k8s-mon获取容器tag时需要适配，默认采…

…用docker-api，失败再尝试containerd-api

* [CHANGE] pod runner改为`yauritux/busybox-curl` 提供curl命令方便排查问题
* [CHANGE] 注意如果 不采集etcd，没有创建对应的证书(如k8s使用公有云托管的)，那么请将 deployment中挂载证书那几行注释掉，不然容器起不来
* [CHANGE] 容器版本调整为 v2.1.0

CHANGELOG.md

@@ -1,3 +1,10 @@
+## v2.1.0 / 2021-04-09
+* [FEATURE] k8s 1.20后续版本默认容器运行时采用`containerd`，k8s-mon获取容器tag时需要适配，默认采用docker-api，失败再尝试containerd-api
+* [CHANGE] pod runner改为`yauritux/busybox-curl` 提供curl命令方便排查问题
+* [CHANGE] 注意如果 不采集etcd，没有创建对应的证书(如k8s使用公有云托管的)，那么请将 deployment中挂载证书那几行注释掉，不然容器起不来
+* [CHANGE] 容器版本调整为 v2.1.0
+
+
 ## v2.0.7 / 2021-03-30
 * [BUGFIX] hold点/预聚合所使用的共享map`dataMap.Map`改为`go-cache` ，用来做gc，避免pod滚动后旧的数据没有删除导致内存不回收
 * [CHANGE] 编译时传入version，便于打印版本信息

Dockerfile

@@ -9,7 +9,8 @@ RUN go mod download
 COPY . .
 RUN  CGO_ENABLED=0 go build -o server  -ldflags "-X 'github.com/prometheus/common/version.BuildUser=root@n9e'  -X 'github.com/prometheus/common/version.BuildDate=`date`' -X 'github.com/prometheus/common/version.Version=`cat VERSION`'"
 #FROM scratch as runner
-FROM busybox  as runner
+#FROM busybox  as runner
+FROM yauritux/busybox-curl  as runner
 COPY --from=builder /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder /usr/src/app/server /opt/app/k8s-mon

VERSION

@@ -1 +1 @@
-2.0.7
+2.1.0

collect/containerd_sdk.go

@@ -0,0 +1,61 @@
+package collect
+
+import (
+	"context"
+	"errors"
+	"github.com/containerd/containerd"
+)
+
+func getLabelMapByContainerdSdk(nidLabelName string) (map[string]map[string]string, error) {
+	client, err := containerd.New("/run/containerd/containerd.sock", containerd.WithDefaultNamespace("k8s.io"))
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer client.Close()
+
+	context := context.Background()
+	cers, err := client.Containers(context)
+	if err != nil {
+		return nil, err
+	}
+	if len(cers) == 0 {
+		return nil, errors.New("got zero containers on this node")
+	}
+
+	insM := make(map[string]map[string]string)
+	whiteKeyM := map[string]struct{}{
+		nidLabelName: {},
+	}
+
+	for _, c := range cers {
+		labels, err := c.Labels(context)
+		if err != nil {
+			continue
+		}
+
+		podName := labels["io.kubernetes.pod.name"]
+		if podName == "" {
+			continue
+		}
+
+		lastM, loaded := insM[podName]
+		if !loaded {
+			lastM = make(map[string]string)
+		}
+
+		for k, v := range labels {
+
+			if _, found := whiteKeyM[k]; !found {
+				continue
+			}
+			lastM[k] = v
+		}
+
+		insM[podName] = lastM
+
+	}
+	return insM, nil
+
+}

collect/kubelet_cadvisor.go

@@ -30,11 +30,20 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,
 		level.Error(logger).Log("msg", "DoKubeletCollectEmptyMetricsResult")
 		return
 	}
-
+	/*
+		1. 优先使用docker-api
+		2. 出错了再用containerd-api尝试下
+		3. 1.20.1以上的k8s默认CONTAINER-RUNTIME 为containerd
+		4. 考虑多个节点可能一部分采用docker 一部分 containerd
+	*/
 	insM, err := getLabelMapByDockerSdk(cg.N9eNidLabelName)
 	if err != nil {
-		level.Error(logger).Log("msg", "DoKubeletCollect getInspectAll error", "err", err)
-		return
+		level.Error(logger).Log("msg", "DoKubeletCollect_getPodMap_by_dockerApi_error_try_containerd...", "err", err)
+		insM, err = getLabelMapByContainerdSdk(cg.N9eNidLabelName)
+		if err != nil {
+			level.Error(logger).Log("msg", "DoKubeletCollect_getPodMap_by_containerdApi_error", "err", err)
+			return
+		}
 	}
 
 	// tag 白名单

k8s-config/daemonSet.yaml

@@ -18,8 +18,8 @@ spec:
       hostNetwork: true
       containers:
         - name: k8s-mon
-          image:  registry.cn-beijing.aliyuncs.com/n9e/k8s-mon:v1
-          #image:  k8s-mon:v1
+          image:  registry.cn-beijing.aliyuncs.com/n9e/k8s-mon:v2.1.0
+#          image:  k8s-mon:v1
           imagePullPolicy: IfNotPresent
           command:
             - /opt/app/k8s-mon
@@ -30,8 +30,15 @@ spec:
               name: config-volume
             - mountPath: /var/run
               name: docker-api
+              # docker inspect 需要方位 unix:///var/run/docker.sock
               # 赋予write权限，因为service account需要
               #readOnly: true
+
+            - mountPath: /run
+              name: containerd-api
+              readOnly: true
+              # 使用containerd作为运行时需要访问 /run/containerd/containerd.sock
+
           resources:
             requests:
               cpu: 100m
@@ -47,4 +54,7 @@ spec:
           name: config-volume
         - name: docker-api
           hostPath:
-            path: /var/run
+            path: /var/run
+        - name: containerd-api
+          hostPath:
+            path: /run

k8s-config/deployment.yaml

@@ -22,7 +22,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: k8s-mon
-          image:  registry.cn-beijing.aliyuncs.com/n9e/k8s-mon:v1
+          image:  registry.cn-beijing.aliyuncs.com/n9e/k8s-mon::v2.1.0
           #image:  k8s-mon:v1
           command:
             - /opt/app/k8s-mon

readme.md

@@ -68,6 +68,7 @@
 # 创建namespace kube-admin
 kubectl create ns kube-admin
 # 创建访问etcd所需secret，在master上执行（不采集etcd则不需要）
+# 注意如果 不采集etcd，没有创建对应的证书(如k8s使用公有云托管的)，那么请将 deployment中挂载证书那几行注释掉，不然容器起不来
 kubectl create secret generic etcd-certs --from-file=/etc/kubernetes/pki/etcd/healthcheck-client.crt --from-file=/etc/kubernetes/pki/etcd/healthcheck-client.key --from-file=/etc/kubernetes/pki/etcd/ca.crt -n kube-admin
 
 ```