eventcache: Refactor retries counter

Use new helpers from pkg/metrics to define custom metrics and collector.

Rename the metric to tetragon_event_cache_fetch_retries_total, to make it clear
what is being retried.

Signed-off-by: Anna Kapuscinska <anna@isovalent.com>

contrib/upgrade-notes/latest.md

@@ -59,3 +59,4 @@ tetragon:
 * `tetragon_event_cache_<entry_type>_errors_total` metrics are replaced by
   `tetragon_event_cache_fetch_failures_total{entry_type="<entry_type>"}`.
 * `tetragon_event_cache_accesses_total` metric is renamed to `tetragon_event_cache_inserts_total`.
+* `tetragon_event_cache_retries_total` metric is renamed to `tetragon_event_cache_fetch_retries_total`.

pkg/eventcache/eventcache.go

@@ -68,7 +68,7 @@ func HandleGenericInternal(ev notify.Event, pid uint32, tid *uint32, timestamp u
 	if parent != nil {
 		ev.SetParent(parent.UnsafeGetProcess())
 	} else {
-		EventCacheRetries(ParentInfo).Inc()
+		CacheRetries(ParentInfo).Inc()
 		err = ErrFailedToGetParentInfo
 	}
 
@@ -84,7 +84,7 @@ func HandleGenericInternal(ev notify.Event, pid uint32, tid *uint32, timestamp u
 		process.UpdateEventProcessTid(proc, tid)
 		ev.SetProcess(proc)
 	} else {
-		EventCacheRetries(ProcessInfo).Inc()
+		CacheRetries(ProcessInfo).Inc()
 		err = ErrFailedToGetProcessInfo
 	}
 
@@ -101,7 +101,7 @@ func HandleGenericInternal(ev notify.Event, pid uint32, tid *uint32, timestamp u
 func HandleGenericEvent(internal *process.ProcessInternal, ev notify.Event, tid *uint32) error {
 	p := internal.UnsafeGetProcess()
 	if option.Config.EnableK8s && p.Pod == nil {
-		EventCacheRetries(PodInfo).Inc()
+		CacheRetries(PodInfo).Inc()
 		return ErrFailedToGetPodInfo
 	}
 

pkg/eventcache/metrics.go

@@ -62,11 +62,6 @@ var (
 		Help:        "The total of errors encountered while fetching process exec information from the cache.",
 		ConstLabels: nil,
 	}, []string{"error", "event_type"})
-	eventCacheRetriesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: consts.MetricsNamespace,
-		Name:      "event_cache_retries_total",
-		Help:      "The total number of retries for event caching per entry type.",
-	}, []string{"entry_type"})
 	cacheSize = metrics.MustNewCustomGauge(metrics.NewOpts(
 		consts.MetricsNamespace, "", "event_cache_entries",
 		"The number of entries in the event cache.",
@@ -79,6 +74,11 @@ var (
 		Help:        "Number of inserts to the event cache.",
 		ConstLabels: nil,
 	})
+	cacheRetries = metrics.MustNewCounter(metrics.NewOpts(
+		consts.MetricsNamespace, subsystem, "fetch_retries_total",
+		"Number of retries when fetching info from the event cache.",
+		nil, []metrics.ConstrainedLabel{entryTypeLabel}, nil,
+	), nil)
 	failedFetches = metrics.MustNewCounter(metrics.NewOpts(
 		consts.MetricsNamespace, subsystem, "fetch_failures_total",
 		"Number of failed fetches from the event cache. These won't be retried as they already exceeded the limit.",
@@ -102,19 +102,16 @@ func newCacheCollector() prometheus.Collector {
 
 func RegisterMetrics(group metrics.Group) {
 	group.MustRegister(eventCacheErrorsTotal)
-	group.MustRegister(eventCacheRetriesTotal)
 	group.MustRegister(
 		newCacheCollector(),
 		cacheInserts,
+		cacheRetries,
 		failedFetches,
 	)
 }
 
 func InitMetrics() {
 	// Initialize metrics with labels
-	for en := range cacheEntryTypeLabelValues {
-		EventCacheRetries(en).Add(0)
-	}
 	for ev := range tetragon.EventType_name {
 		if tetragon.EventType(ev) != tetragon.EventType_UNDEF && tetragon.EventType(ev) != tetragon.EventType_TEST {
 			for er := range cacheErrorLabelValues {
@@ -130,6 +127,6 @@ func EventCacheError(er CacheError, eventType tetragon.EventType) prometheus.Cou
 }
 
 // Get a new handle on an eventCacheRetriesTotal metric for an entryType
-func EventCacheRetries(entryType CacheEntryType) prometheus.Counter {
-	return eventCacheRetriesTotal.WithLabelValues(entryType.String())
+func CacheRetries(entryType CacheEntryType) prometheus.Counter {
+	return cacheRetries.WithLabelValues(entryType.String())
 }

pkg/grpc/exec/exec.go

@@ -191,7 +191,7 @@ func (msg *MsgExecveEventUnix) Retry(internal *process.ProcessInternal, ev notif
 		cgroupID := msg.Unix.Kube.Cgrpid
 		podInfo = process.GetPodInfo(cgroupID, containerId, filename, args, nspid)
 		if podInfo == nil {
-			eventcache.EventCacheRetries(eventcache.PodInfo).Inc()
+			eventcache.CacheRetries(eventcache.PodInfo).Inc()
 			return eventcache.ErrFailedToGetPodInfo
 		}
 	}
@@ -420,7 +420,7 @@ func (msg *MsgExitEventUnix) RetryInternal(ev notify.Event, timestamp uint64) (*
 			msg.RefCntDone[ParentRefCnt] = true
 		}
 	} else {
-		eventcache.EventCacheRetries(eventcache.ParentInfo).Inc()
+		eventcache.CacheRetries(eventcache.ParentInfo).Inc()
 		err = eventcache.ErrFailedToGetParentInfo
 	}
 
@@ -432,7 +432,7 @@ func (msg *MsgExitEventUnix) RetryInternal(ev notify.Event, timestamp uint64) (*
 			msg.RefCntDone[ProcessRefCnt] = true
 		}
 	} else {
-		eventcache.EventCacheRetries(eventcache.ProcessInfo).Inc()
+		eventcache.CacheRetries(eventcache.ProcessInfo).Inc()
 		err = eventcache.ErrFailedToGetProcessInfo
 	}
 
@@ -486,7 +486,7 @@ func (msg *MsgProcessCleanupEventUnix) RetryInternal(_ notify.Event, timestamp u
 			msg.RefCntDone[ParentRefCnt] = true
 		}
 	} else {
-		eventcache.EventCacheRetries(eventcache.ParentInfo).Inc()
+		eventcache.CacheRetries(eventcache.ParentInfo).Inc()
 		err = eventcache.ErrFailedToGetParentInfo
 	}
 
@@ -496,7 +496,7 @@ func (msg *MsgProcessCleanupEventUnix) RetryInternal(_ notify.Event, timestamp u
 			msg.RefCntDone[ProcessRefCnt] = true
 		}
 	} else {
-		eventcache.EventCacheRetries(eventcache.ProcessInfo).Inc()
+		eventcache.CacheRetries(eventcache.ProcessInfo).Inc()
 		err = eventcache.ErrFailedToGetProcessInfo
 	}