diff --git a/cmd/tetra/metrics/print.go b/cmd/tetra/metrics/print.go index acc2a910df3..77d76134055 100644 --- a/cmd/tetra/metrics/print.go +++ b/cmd/tetra/metrics/print.go @@ -57,6 +57,6 @@ func New() *cobra.Command { } func initMetrics(_ string, reg *prometheus.Registry, _ *slog.Logger) error { - metricsconfig.InitAllMetrics(reg) + metricsconfig.InitAllMetricsForDocs(reg) return nil } diff --git a/pkg/metrics/bpfmetric.go b/pkg/metrics/bpfmetric.go index 3a660e63eb9..8e0d1bb0367 100644 --- a/pkg/metrics/bpfmetric.go +++ b/pkg/metrics/bpfmetric.go @@ -3,7 +3,9 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/prometheus/client_golang/prometheus" +) // BPFMetric represents a metric read directly from a BPF map. // It's intended to be used in custom collectors. The interface doesn't provide @@ -16,13 +18,30 @@ type BPFMetric interface { type bpfCounter struct { desc *prometheus.Desc + prometheus.CounterOpts + VariableLabels []string } func NewBPFCounter(desc *prometheus.Desc) BPFMetric { return &bpfCounter{desc: desc} } +func NewBPFCounterFromOpts(opts prometheus.CounterOpts, labelNames []string) *bpfCounter { + return &bpfCounter{ + CounterOpts: opts, + VariableLabels: labelNames, + } +} + func (c *bpfCounter) Desc() *prometheus.Desc { + if c.desc == nil { + c.desc = prometheus.NewDesc( + prometheus.BuildFQName(c.Namespace, c.Subsystem, c.Name), + c.Help, + c.VariableLabels, + c.ConstLabels, + ) + } return c.desc } @@ -30,6 +49,15 @@ func (c *bpfCounter) MustMetric(value float64, labelValues ...string) prometheus return prometheus.MustNewConstMetric(c.desc, prometheus.CounterValue, value, labelValues...) } +func (c *bpfCounter) ToProm() *prometheus.CounterVec { + return prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: c.Namespace, + Name: c.Name, + Help: c.Help, + ConstLabels: c.ConstLabels, + }, c.VariableLabels) +} + type bpfGauge struct { desc *prometheus.Desc } diff --git a/pkg/metrics/consts/consts.go b/pkg/metrics/consts/consts.go index c86dea12d09..247b44b257b 100644 --- a/pkg/metrics/consts/consts.go +++ b/pkg/metrics/consts/consts.go @@ -6,3 +6,11 @@ package consts const MetricsNamespace = "tetragon" var KnownMetricLabelFilters = []string{"namespace", "workload", "pod", "binary"} + +var ( + EmptyPolicyHookLabels = []string{"", ""} + EmptyProcessLabels = []string{"", "", "", ""} + + ExamplePolicyHookLabels = []string{"example-tracingpolicy", "example-kprobe"} + ExampleProcessLabels = []string{"example-namespace", "example-workload", "example-pod", "example-binary"} +) diff --git a/pkg/metrics/eventmetrics/eventmetrics.go b/pkg/metrics/eventmetrics/eventmetrics.go index 7ab16433ee3..84ee1ead2f2 100644 --- a/pkg/metrics/eventmetrics/eventmetrics.go +++ b/pkg/metrics/eventmetrics/eventmetrics.go @@ -4,8 +4,12 @@ package eventmetrics import ( + "fmt" + "slices" + "github.com/cilium/tetragon/api/v1/tetragon" "github.com/cilium/tetragon/api/v1/tetragon/codegen/helpers" + "github.com/cilium/tetragon/pkg/api/ops" "github.com/cilium/tetragon/pkg/api/processapi" "github.com/cilium/tetragon/pkg/filters" "github.com/cilium/tetragon/pkg/logger" @@ -26,11 +30,17 @@ var ( Help: "The total number of Tetragon events", ConstLabels: nil, }, []string{"type"}) - MissedEvents = metrics.NewBPFCounter(prometheus.NewDesc( - prometheus.BuildFQName(consts.MetricsNamespace, "", "missed_events_total"), - "The total number of Tetragon events per type that are failed to sent from the kernel.", - []string{"msg_op"}, nil, - )) + // MissedEvents = metrics.NewBPFCounter(prometheus.NewDesc( + // prometheus.BuildFQName(consts.MetricsNamespace, "", "missed_events_total"), + // "The total number of Tetragon events per type that are failed to sent from the kernel.", + // []string{"msg_op"}, nil, + // )) + MissedEvents = metrics.NewBPFCounterFromOpts(prometheus.CounterOpts{ + Namespace: consts.MetricsNamespace, + Name: "flagsmissed_events_total_total", + Help: "The total number of Tetragon events per type that are failed to sent from the kernel.", + ConstLabels: nil, + }, []string{"msg_op"}) FlagCount = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: consts.MetricsNamespace, Name: "flags_total", @@ -63,11 +73,31 @@ func InitMetrics(registry *prometheus.Registry) { registerMetrics(registry) // Initialize metrics with labels - // TODO: - // events_total: list all possible event types, add example process metadata for docs - // missed_events_total: list all reasonable opcodes for docs - // flags_total: list all possible flags - // policy_events_total: add example policy and hook for docs + for ev := range tetragon.EventType_name { + if tetragon.EventType(ev) != tetragon.EventType_UNDEF && tetragon.EventType(ev) != tetragon.EventType_TEST { + EventsProcessed.WithLabelValues(slices.Concat([]string{tetragon.EventType(ev).String()}, consts.EmptyProcessLabels)...).Add(0) + } + } + for _, v := range exec.FlagStrings { + FlagCount.WithLabelValues(v).Add(0) + } + policyStats.WithLabelValues(slices.Concat(consts.EmptyPolicyHookLabels, consts.EmptyProcessLabels)...).Add(0) +} + +func InitMetricsForDocs(registry *prometheus.Registry) { + InitMetrics(registry) + for ev := range tetragon.EventType_name { + if tetragon.EventType(ev) != tetragon.EventType_UNDEF && tetragon.EventType(ev) != tetragon.EventType_TEST { + EventsProcessed.WithLabelValues(slices.Concat([]string{tetragon.EventType(ev).String()}, consts.ExampleProcessLabels)...).Add(0) + } + } + registry.MustRegister(MissedEvents.ToProm()) + for opcode := range ops.OpCodeStrings { + if opcode != ops.MsgOpUndef && opcode != ops.MsgOpTest { + MissedEvents.ToProm().WithLabelValues(fmt.Sprint(int32(opcode))).Add(0) + } + } + policyStats.WithLabelValues(slices.Concat(consts.ExamplePolicyHookLabels, consts.ExampleProcessLabels)...).Add(0) } func GetProcessInfo(process *tetragon.Process) (binary, pod, workload, namespace string) { diff --git a/pkg/metrics/metricsconfig/initmetrics.go b/pkg/metrics/metricsconfig/initmetrics.go index 46024c41005..7a2861bdc55 100644 --- a/pkg/metrics/metricsconfig/initmetrics.go +++ b/pkg/metrics/metricsconfig/initmetrics.go @@ -60,3 +60,36 @@ func InitAllMetrics(registry *prometheus.Registry) { registry.MustRegister(grpcmetrics.NewServerMetrics()) version.InitMetrics(registry) } + +func InitAllMetricsForDocs(registry *prometheus.Registry) { + errormetrics.InitMetrics(registry) + eventcachemetrics.InitMetrics(registry) + eventmetrics.InitMetricsForDocs(registry) + kprobemetrics.InitMetrics(registry) + mapmetrics.InitMetrics(registry) + opcodemetrics.InitMetrics(registry) + policyfiltermetrics.InitMetrics(registry) + processexecmetrics.InitMetrics(registry) + ringbufmetrics.InitMetrics(registry) + ringbufqueuemetrics.InitMetrics(registry) + syscallmetrics.InitMetrics(registry) + watchermetrics.InitMetrics(registry) + observer.InitMetrics(registry) + tracing.InitMetrics(registry) + ratelimitmetrics.InitMetrics(registry) + policystatemetrics.InitMetrics(registry) + + // register BPF collectors + registry.MustRegister(mapmetrics.NewBPFCollector( + eventcache.NewBPFCollector(), + observer.NewBPFCollector(), + process.NewBPFCollector(), + )) + // registry.MustRegister(eventmetrics.NewBPFCollector()) + + // register common third-party collectors + registry.MustRegister(collectors.NewGoCollector()) + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + registry.MustRegister(grpcmetrics.NewServerMetrics()) + version.InitMetrics(registry) +}