diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml index 839a8dc0..d4b76e0d 100644 --- a/deploy/helm/Chart.yaml +++ b/deploy/helm/Chart.yaml @@ -1,16 +1,15 @@ apiVersion: v1 -name: kubeskoop-exporter +name: kubeskoop description: Promtheus exporter and kernel tracing for OS metrics in kubernetes, with pluggable metric collectors and kernel eBPF tracers. type: application annotations: category: Analytics licenses: Apache-2.0 -version: 0.3.0 -appVersion: 0.1.0 +version: 1.0.0 +appVersion: 1.0.0 icon: https://img.shields.io/github/v/tag/alibaba/kubeskoop keywords: - kubeskoop - - kubeskoop-exporter - kubernetes - prometheus - eBPF diff --git a/deploy/helm/templates/configMap.yaml b/deploy/helm/templates/configMap.yaml index d6675941..e728c375 100644 --- a/deploy/helm/templates/configMap.yaml +++ b/deploy/helm/templates/configMap.yaml @@ -1,17 +1,17 @@ apiVersion: v1 -data: - config.yaml: |- - debugMode: {{ .Values.debugMode }} - port: {{ .Values.config.serverPort }} - metrics: - probes:{{- range .Values.config.metricsProbes }} - - {{ . }}{{- end }} - events: - probes: {{- range .Values.config.eventProbes }} - - {{ . }}{{- end }} - sinks: {{- range .Values.config.eventSinks }} - - {{ . }}{{- end }} kind: ConfigMap metadata: name: kubeskoop-config namespace: {{ .Release.Namespace }} +data: + config.yaml: |- + debugmode: {{ .Values.agent.debug }} + port: {{ .Values.agent.port }} + metrics: + probes: + {{- toYaml .Values.config.metricProbes | nindent 6 }} + event: + probes: + {{- toYaml .Values.config.eventProbes | nindent 6 }} + sinks: + {{- toYaml .Values.config.eventSinks | nindent 6 }} diff --git a/deploy/helm/templates/controller/clusterrole.yaml b/deploy/helm/templates/controller/clusterrole.yaml new file mode 100644 index 00000000..bab1d270 --- /dev/null +++ b/deploy/helm/templates/controller/clusterrole.yaml @@ -0,0 +1,28 @@ +{{- if .Values.controller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeskoop-controller +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - configmaps + - namespaces + - pods + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["get", "list"] +- apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "list"] +- apiGroups: ["projectcalico.org", "crd.projectcalico.org"] + resources: ["ippools"] + verbs: ["get", "list"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} \ No newline at end of file diff --git a/deploy/helm/templates/controller/clusterrolebinding.yaml b/deploy/helm/templates/controller/clusterrolebinding.yaml new file mode 100644 index 00000000..27f31689 --- /dev/null +++ b/deploy/helm/templates/controller/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +{{- if .Values.controller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeskoop-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeskoop-controller +subjects: +- kind: ServiceAccount + name: default + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/deploy/helm/templates/controller/configmap.yaml b/deploy/helm/templates/controller/configmap.yaml new file mode 100644 index 00000000..a1a87951 --- /dev/null +++ b/deploy/helm/templates/controller/configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.controller.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: controller-config + namespace: {{ .Release.Namespace }} +data: + controller.yaml: |- + logLevel: debug + server: + httpPort: 10264 + agentPort: 10263 + controller: + namespace: {{ .Release.Namespace }} + prometheus: "{{ .Values.controller.config.prometheusEndpoint }}" + loki: "{{ .Values.controller.config.lokiEndpoint }}" + database: + type: sqlite3 + diagnose: {} + {{- end }} \ No newline at end of file diff --git a/deploy/helm/templates/controller/deployment.yaml b/deploy/helm/templates/controller/deployment.yaml new file mode 100644 index 00000000..bd093120 --- /dev/null +++ b/deploy/helm/templates/controller/deployment.yaml @@ -0,0 +1,47 @@ +{{- if .Values.controller.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller + namespace: {{ .Release.Namespace }} +{{- with .Values.controller }} +spec: + replicas: 1 + selector: + matchLabels: + app: controller + template: + metadata: + name: controller + labels: + app: controller + spec: + containers: + - name: controller + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.imagePullPolicy }} + command: + - "/bin/controller" + volumeMounts: + - name: lib + mountPath: /var/lib/kubeskoop + - name: config + mountPath: /etc/kubeskoop + resources: + {{ toYaml .resources | nindent 12 }} + {{- with .nodeSelector }} + nodeSelector: + {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .tolerations }} + tolerations: + {{ toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: lib + emptyDir: { } + - name: config + configMap: + name: controller-config +{{- end }} +{{- end }} \ No newline at end of file diff --git a/deploy/helm/templates/controller/role.yaml b/deploy/helm/templates/controller/role.yaml new file mode 100644 index 00000000..5f50ffd3 --- /dev/null +++ b/deploy/helm/templates/controller/role.yaml @@ -0,0 +1,19 @@ +{{- if .Values.controller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: controller + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubeskoop-config"] + verbs: ["get", "update"] +- apiGroups: [""] + resources: + - pods + verbs: ["get", "list", "watch", "delete", "create"] +- apiGroups: [""] + resources: ["pods/exec", "pods/attach", "pods/portforward"] + verbs: ["create", "get", "list", "update", "delete"] +{{- end }} \ No newline at end of file diff --git a/deploy/helm/templates/controller/rolebinding.yaml b/deploy/helm/templates/controller/rolebinding.yaml new file mode 100644 index 00000000..7922d727 --- /dev/null +++ b/deploy/helm/templates/controller/rolebinding.yaml @@ -0,0 +1,15 @@ +{{- if .Values.controller.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: controller + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: controller +subjects: +- kind: ServiceAccount + name: default + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/deploy/helm/templates/controller/service.yaml b/deploy/helm/templates/controller/service.yaml new file mode 100644 index 00000000..e38a2f08 --- /dev/null +++ b/deploy/helm/templates/controller/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.controller.enabled}} +apiVersion: v1 +kind: Service +metadata: + name: controller + namespace: {{ .Release.Namespace }} +spec: + selector: + app: controller + ports: + - name: grpc + port: 10263 + targetPort: 10263 + - name: http + port: 10264 + targetPort: 10264 +{{- end }} \ No newline at end of file diff --git a/deploy/helm/templates/deamonSet.yaml b/deploy/helm/templates/daemonset.yaml similarity index 77% rename from deploy/helm/templates/deamonSet.yaml rename to deploy/helm/templates/daemonset.yaml index ebadc19f..a0673ce2 100644 --- a/deploy/helm/templates/deamonSet.yaml +++ b/deploy/helm/templates/daemonset.yaml @@ -1,34 +1,34 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ .Values.name }} + name: agent namespace: {{ .Release.Namespace }} labels: - app: {{ .Values.appName }} + app: kubeskoop-agent +{{- with .Values.agent }} spec: selector: matchLabels: - app: {{ .Values.appName }} + app: kubeskoop-agent template: metadata: labels: - app: {{ .Values.appName }} + app: kubeskoop-agent annotations: prometheus.io/path: /metrics - prometheus.io/port: "9102" + prometheus.io/port: "{{ .config.port }}" prometheus.io/scheme: http prometheus.io/scrape: "true" - name: {{ .Values.appName }} spec: hostNetwork: true hostPID: true hostIPC: true dnsPolicy: ClusterFirstWithHostNet - {{- if .Values.initContainer.enabled }} + {{- if .btfhack.enabled }} initContainers: - - name: preparels - image: {{ .Values.initContainer.repository }}:{{ .Values.initContainer.tag }} - imagePullPolicy: {{ .Values.initContainer.imagePullPolicy }} + - name: btfhack + image: "{{ .btfhack.repository }}:{{ .btfhack.tag }}" + imagePullPolicy: {{ .btfhack.imagePullPolicy }} volumeMounts: - name: bpfdir mountPath: /etc/net-exporter/btf @@ -42,8 +42,8 @@ spec: {{- end }} containers: - name: inspector - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.imagePullPolicy }} + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.imagePullPolicy }} env: - name: INSPECTOR_NODENAME valueFrom: @@ -74,12 +74,12 @@ spec: securityContext: privileged: true resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml .resources | nindent 12 }} + {{- with .nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with .tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} @@ -89,7 +89,7 @@ spec: path: /proc - name: runtimeendpoint hostPath: - path: {{ .Values.runtimeEndpoint }} + path: {{ .config.runtimeEndpoint }} - name: boot hostPath: path: /boot/ @@ -112,3 +112,4 @@ spec: name: kubeskoop-config - name: bpfdir emptyDir: {} +{{- end }} diff --git a/deploy/helm/templates/webconsole/deployment.yaml b/deploy/helm/templates/webconsole/deployment.yaml new file mode 100644 index 00000000..89a30515 --- /dev/null +++ b/deploy/helm/templates/webconsole/deployment.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.controller.enabled .Values.webconsole.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: webconsole + namespace: {{ .Release.Namespace }} +{{- with .Values.webconsole }} +spec: + replicas: 1 + selector: + matchLabels: + app: webconsole + template: + metadata: + name: webconsole + labels: + app: webconsole + spec: + containers: + - name: webconsole + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.imagePullPolicy }} + command: [ "/bin/webconsole" ] + env: + - name: CONTROLLER_ENDPOINT + value: "http://controller:10264" + - name: GRAFANA_PROXY + value: "{{ .grafana.proxy }}" + - name: GRAFANA_ENDPOINT + value: "{{ .grafana.endpoint }}" + - name: GRAFANA_USERNAME + value: "{{ .grafana.username }}" + - name: GRAFANA_PASSWORD + value: "{{ .grafana.password }}" + - name: AUTH_USERNAME + value: "{{ .auth.username }}" + - name: AUTH_PASSWORD + value: "{{ .auth.password }}" + resources: + {{- toYaml .resources | nindent 12 }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + {{ with .nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{ with .tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deploy/helm/templates/webconsole/service.yaml b/deploy/helm/templates/webconsole/service.yaml new file mode 100644 index 00000000..80f41fba --- /dev/null +++ b/deploy/helm/templates/webconsole/service.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.controller.enabled .Values.webconsole.enabled}} +apiVersion: v1 +kind: Service +metadata: + name: webconsole + namespace: {{ .Release.Namespace }} + annotations: + {{- .Values.webconsole.service.annotations | toYaml | nindent 4 }} +spec: + type: {{ .Values.webconsole.service.type }} + selector: + app: webconsole + ports: + - name: http + port: {{ .Values.webconsole.service.port }} + targetPort: 8080 +{{- end }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index deb9c165..f15ac9ba 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1,49 +1,100 @@ -name: kubeskoop-exporter -debugMode: false -runtimeEndpoint: /run/containerd/containerd.sock -appName: kubeskoop-exporter - -image: - repository: kubeskoop/agent - # Overrides the image tag whose default is the chart appVersion. - tag: v1.0.0 - imagePullPolicy: IfNotPresent - -initContainer: - enabled: true - repository: kubeskoop/agent - tag: v1.0.0 - imagePullPolicy: IfNotPresent - -resources: - limits: - cpu: 500m - memory: 1024Mi - requests: - cpu: 500m - memory: 1024Mi - config: - serverPort: 9102 - metricsProbes: + metricProbes: + - name: conntrack + - name: qdisc - name: netdev - name: io - - name: socketlatency - - name: packetloss - - name: softirq - - name: tcpext + - name: sock - name: tcpsummary - name: tcp - - name: sock - - name: softnet + - name: tcpext - name: udp - - name: virtcmdlatency - name: kernellatency + - name: packetloss + - name: flow + args: + enablePortInLabel: false + - name: tcpretrans eventProbes: - - name: tcpreset + - name: biolatency + - name: kernellatency - name: packetloss + args: + enableStack: false + - name: tcpreset + - name: tcpretrans eventSinks: - name: stderr -nodeSelector: { } -tolerations: { } +agent: + config: + debug: false + port: 9102 + runtimeEndpoint: /run/containerd/containerd.sock + image: + repository: kubeskoop/agent + tag: v1.0.0 + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 500m + memory: 1024Mi + requests: + cpu: 500m + memory: 1024Mi + btfhack: + enabled: true + repository: kubeskoop/agent + tag: v1.0.0 + imagePullPolicy: IfNotPresent + nodeSelector: {} + tolerations: {} + +controller: + enabled: true + config: + logLevel: info + prometheusEndpoint: http://prometheus-service + lokiEndpoint: http://loki-service:3100 + image: + repository: kubeskoop/controller + tag: v1.0.0 + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 500m + memory: 200Mi + requests: + cpu: 50m + memory: 20Mi + nodeSelector: {} + tolerations: {} + +webconsole: + enabled: true + service: + type: NodePort + port: 80 + annotations: {} + auth: + username: admin + password: kubeskoop + grafana: + endpoint: http://grafana/grafana + proxy: true + # used for proxy mode + username: admin + password: kubeskoop + image: + repository: kubeskoop/controller + tag: v1.0.0 + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 500m + memory: 200Mi + requests: + cpu: 50m + memory: 20Mi + nodeSelector: { } + tolerations: { } diff --git a/pkg/controller/service/controller.go b/pkg/controller/service/controller.go index e59bc349..0da79c27 100644 --- a/pkg/controller/service/controller.go +++ b/pkg/controller/service/controller.go @@ -3,7 +3,6 @@ package service import ( "context" "io" - "os" "sync" "time" @@ -45,17 +44,23 @@ type ControllerService interface { } type Config struct { - KubeConfig string + Namespace string `yaml:"namespace"` + KubeConfig string `yaml:"kubeConfig"` Prometheus string `yaml:"prometheus"` + Loki string `yaml:"loki"` DB db.Config `yaml:"database"` Diagnose diagnose.Config `yaml:"diagnose"` } func NewControllerService(k8sClient *kubernetes.Clientset, config *Config) (ControllerService, error) { + if config.Namespace == "" { + config.Namespace = Namespace + } + ctrl := &controller{ taskWatcher: sync.Map{}, resultWatchers: sync.Map{}, - Namespace: Namespace, + Namespace: config.Namespace, ConfigMapName: ExporterConfigMap, } @@ -76,8 +81,8 @@ func NewControllerService(k8sClient *kubernetes.Clientset, config *Config) (Cont ctrl.promClient = promClient } - if lokiEndpoint, ok := os.LookupEnv("LOKI_ENDPOINT"); ok { - lokiClient, err := lokiwrapper.NewClient(lokiEndpoint) + if config.Loki != "" { + lokiClient, err := lokiwrapper.NewClient(config.Loki) if err != nil { return nil, err }