feat(metrics): add additional labels to metrics

alibaba · May 9, 2024 · 73dacb2 · 73dacb2
1 parent 63a7471
commit 73dacb2
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 4 deletions.
diff --git a/pkg/exporter/cmd/config.go b/pkg/exporter/cmd/config.go
@@ -16,7 +16,8 @@ type InspServerConfig struct {
 }
 
 type MetricsConfig struct {
-	Probes []ProbeConfig `yaml:"probes" mapstructure:"probes" json:"probes"`
+	Probes           []ProbeConfig `yaml:"probes" mapstructure:"probes" json:"probes"`
+	AdditionalLabels string        `yaml:"additionalLabels" mapstructure:"additionalLabels" json:"additionalLabels"`
 }
 
 type EventConfig struct {

diff --git a/pkg/exporter/cmd/server.go b/pkg/exporter/cmd/server.go
@@ -289,6 +289,12 @@ func (i *inspServer) start(cfg *InspServerConfig) error {
 		var err error
 		ctx := context.TODO()
 
+		err = probe.InitAdditionalLabels(cfg.MetricsConfig.AdditionalLabels)
+		if err != nil {
+			log.Errorf("failed init additional labels: %v", err)
+			return
+		}
+
 		log.Infof("start metrics server")
 		i.metricsServer, err = NewMetricsServer()
 		if err != nil {

diff --git a/pkg/exporter/nettop/cache.go b/pkg/exporter/nettop/cache.go
@@ -163,6 +163,7 @@ type Entity struct {
 	podMeta
 	initPid int
 	pids    []int
+	labels  map[string]string
 }
 
 func (e *Entity) String() string {
@@ -177,6 +178,10 @@ func (e *Entity) GetPodNamespace() string {
 	return e.podMeta.namespace
 }
 
+func (e *Entity) GetLabels() map[string]string {
+	return e.labels
+}
+
 func (e *Entity) IsHostNetwork() bool {
 	return e.netnsMeta.isHostNetwork
 }
@@ -353,6 +358,7 @@ func cacheNetTopology(ctx context.Context) error {
 
 		namespace := sandbox.Metadata.Namespace
 		name := sandbox.Metadata.Name
+		labels := sandbox.Labels
 
 		sandboxStatus, err := criClient.PodSandboxStatus(sandbox.Id, true)
 		if err != nil {
@@ -411,6 +417,7 @@ func cacheNetTopology(ctx context.Context) error {
 				namespace: namespace,
 			},
 			initPid: info.Pid,
+			labels:  labels,
 			pids:    pids,
 		}
 

diff --git a/pkg/exporter/probe/legacy.go b/pkg/exporter/probe/legacy.go
@@ -2,6 +2,7 @@ package probe
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/alibaba/kubeskoop/pkg/exporter/bpfutil"
 	"github.com/alibaba/kubeskoop/pkg/exporter/nettop"
@@ -12,9 +13,63 @@ import (
 var legacyMetricsLabels = []string{"target_node", "target_namespace", "target_pod", "node", "namespace", "pod"}
 var StandardMetricsLabels = []string{"k8s_node", "k8s_namespace", "k8s_pod"}
 var TupleMetricsLabels = []string{"protocol", "src", "src_type", "src_node", "src_namespace", "src_pod", "dst", "dst_type", "dst_node", "dst_namespace", "dst_pod", "sport", "dport"}
+var AdditionalLabelValueExpr []string
 
 func BuildStandardMetricsLabelValues(entity *nettop.Entity) []string {
-	return []string{nettop.GetNodeName(), entity.GetPodNamespace(), entity.GetPodName()}
+	metaPodLabels := []string{nettop.GetNodeName(), entity.GetPodNamespace(), entity.GetPodName()}
+	return append(metaPodLabels, BuildAdditionalLabelsValues(entity)...)
+}
+
+func InitAdditionalLabels(additionalLabels string) error {
+	if len(additionalLabels) == 0 {
+		return nil
+	}
+
+	//split use space as separator
+	additionalLabelsArray := strings.Split(additionalLabels, " ")
+	//append to StandardMetricsLabels and AdditionalLabelValueExpr
+	for additionalKV := range additionalLabelsArray {
+		labelKVPair := strings.Split(additionalLabelsArray[additionalKV], "=")
+		StandardMetricsLabels = append(StandardMetricsLabels, strings.TrimSpace(labelKVPair[0]))
+		AdditionalLabelValueExpr = append(AdditionalLabelValueExpr, strings.TrimSpace(labelKVPair[1]))
+	}
+
+	return nil
+}
+
+func BuildAdditionalLabelsValues(entity *nettop.Entity) []string {
+	if len(AdditionalLabelValueExpr) == 0 {
+		return []string{}
+	}
+
+	var values []string
+	podLabels := entity.GetLabels()
+	for _, labelValue := range AdditionalLabelValueExpr {
+		if strings.Contains(labelValue, "${") && strings.Contains(labelValue, "}") {
+
+			//get the first ${ and last } string in labelValue
+			s := labelValue[strings.Index(labelValue, "${")+2 : strings.LastIndex(labelValue, "}")]
+			// get the key in ${key:value}
+			key := s[strings.Index(s, ":")+1:]
+			// get the value from podLabels
+			value, ok := podLabels[key]
+
+			if !ok {
+				values = append(values, "")
+			} else {
+				// get the prefix and suffix of labelValue
+				prefix := labelValue[:strings.Index(labelValue, "${")]
+				suffix := labelValue[strings.LastIndex(labelValue, "}")+1:]
+
+				values = append(values, prefix+value+suffix)
+			}
+
+		} else {
+			values = append(values, labelValue)
+		}
+	}
+
+	return values
 }
 
 type legacyBatchMetrics struct {
@@ -89,11 +144,12 @@ func (l *legacyBatchMetrics) Collect(metrics chan<- prometheus.Metric) {
 			if err != nil || et == nil {
 				continue
 			}
-			labelValues := []string{nettop.GetNodeName(), et.GetPodNamespace(), et.GetPodName()}
+			labelValues := BuildStandardMetricsLabelValues(et)
 			// for legacy pod labels
 			emit(newMetricsName(l.module, key), labelValues, float64(value))
 
-			labelValues = append(labelValues, labelValues...)
+			var metaPodData = []string{nettop.GetNodeName(), et.GetPodNamespace(), et.GetPodName()}
+			labelValues = append(metaPodData, metaPodData...)
 			emit(legacyMetricsName(l.module, key, l.underscore), labelValues, float64(value))
 		}
 	}