frontend: refactor metrics

This commit refactors the frontend service to report more sensible and
useful metrics.

The visible changes are:
* The `endpoint` label of the `frontend_health` gauge metric is dropped.
* The `frontend_requests_total` gauge metric is renamed to
  `frontend_http_requests_total` (counter).
* The `frontend_duration` gauge metric is renamed to
  `frontend_http_requests_duration_seconds` (histogram).

The histogram buckets have been chosen to match the MSFT requirements in
terms of latency: [0.25s, 0.5s, 1s, 2.5s, 5s, 10s].

Signed-off-by: Simon Pasquier <spasquie@redhat.com>

frontend/cmd/cmd.go

@@ -109,9 +109,6 @@ func (opts *FrontendOpts) Run() error {
 	logger := util.DefaultLogger()
 	logger.Info(fmt.Sprintf("%s (%s) started", frontend.ProgramName, util.Version()))
 
-	// Initialize the Prometheus emitter.
-	prometheusEmitter := frontend.NewPrometheusEmitter(prometheus.DefaultRegisterer)
-
 	// Initialize the global OpenTelemetry tracer.
 	otelShutdown, err := frontend.ConfigureOpenTelemetryTracer(ctx, logger, semconv.CloudRegion(opts.location))
 	if err != nil {
@@ -179,7 +176,7 @@ func (opts *FrontendOpts) Run() error {
 	}
 	logger.Info(fmt.Sprintf("Application running in %s", opts.location))
 
-	f := frontend.NewFrontend(logger, listener, metricsListener, prometheusEmitter, dbClient, opts.location, &csClient)
+	f := frontend.NewFrontend(logger, listener, metricsListener, prometheus.DefaultRegisterer, dbClient, opts.location, &csClient)
 
 	stop := make(chan struct{})
 	signalChannel := make(chan os.Signal, 1)

frontend/go.mod

@@ -17,7 +17,6 @@ require (
 	go.opentelemetry.io/otel/sdk v1.34.0
 	go.opentelemetry.io/otel/trace v1.34.0
 	go.uber.org/mock v0.5.0
-	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
 	golang.org/x/sync v0.11.0
 )
 
@@ -86,7 +85,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/openshift/api v0.0.0-20240429104249-ac9356ba1784
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/client_model v0.6.1
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect

frontend/go.sum

@@ -228,8 +228,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
 golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
-golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY=
-golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

frontend/pkg/frontend/context.go

@@ -15,17 +15,43 @@ import (
 
 type ContextError struct {
 	got any
+	key contextKey
 }
 
 func (c *ContextError) Error() string {
 	return fmt.Sprintf(
-		"error retrieving value from context, value obtained was '%v' and type obtained was '%T'",
+		"error retrieving value for key %q from context, value obtained was '%v' and type obtained was '%T'",
+		c.key,
 		c.got,
 		c.got)
 }
 
 type contextKey int
 
+func (c contextKey) String() string {
+	switch c {
+	case contextKeyOriginalPath:
+		return "originalPath"
+	case contextKeyBody:
+		return "body"
+	case contextKeyLogger:
+		return "logger"
+	case contextKeyVersion:
+		return "version"
+	case contextKeyDBClient:
+		return "dbClient"
+	case contextKeyResourceID:
+		return "resourceID"
+	case contextKeyCorrelationData:
+		return "correlationData"
+	case contextKeySystemData:
+		return "systemData"
+	case contextKeyPattern:
+		return "pattern"
+	}
+	return "<unknown>"
+}
+
 const (
 	// Keys for request-scoped data in http.Request contexts
 	contextKeyOriginalPath contextKey = iota
@@ -36,6 +62,7 @@ const (
 	contextKeyResourceID
 	contextKeyCorrelationData
 	contextKeySystemData
+	contextKeyPattern
 )
 
 func ContextWithOriginalPath(ctx context.Context, originalPath string) context.Context {
@@ -47,6 +74,7 @@ func OriginalPathFromContext(ctx context.Context) (string, error) {
 	if !ok {
 		err := &ContextError{
 			got: originalPath,
+			key: contextKeyOriginalPath,
 		}
 		return originalPath, err
 	}
@@ -62,6 +90,7 @@ func BodyFromContext(ctx context.Context) ([]byte, error) {
 	if !ok {
 		err := &ContextError{
 			got: body,
+			key: contextKeyBody,
 		}
 		return body, err
 	}
@@ -77,6 +106,7 @@ func LoggerFromContext(ctx context.Context) *slog.Logger {
 	if !ok {
 		err := &ContextError{
 			got: logger,
+			key: contextKeyLogger,
 		}
 		// Return the default logger as a fail-safe, but log
 		// the failure to obtain the logger from the context.
@@ -95,6 +125,7 @@ func VersionFromContext(ctx context.Context) (api.Version, error) {
 	if !ok {
 		err := &ContextError{
 			got: version,
+			key: contextKeyVersion,
 		}
 		return version, err
 	}
@@ -110,6 +141,7 @@ func DBClientFromContext(ctx context.Context) (database.DBClient, error) {
 	if !ok {
 		err := &ContextError{
 			got: dbClient,
+			key: contextKeyDBClient,
 		}
 		return dbClient, err
 	}
@@ -125,6 +157,7 @@ func ResourceIDFromContext(ctx context.Context) (*azcorearm.ResourceID, error) {
 	if !ok {
 		err := &ContextError{
 			got: resourceID,
+			key: contextKeyResourceID,
 		}
 		return resourceID, err
 	}
@@ -140,6 +173,7 @@ func CorrelationDataFromContext(ctx context.Context) (*arm.CorrelationData, erro
 	if !ok {
 		err := &ContextError{
 			got: correlationData,
+			key: contextKeyCorrelationData,
 		}
 		return correlationData, err
 	}
@@ -155,8 +189,18 @@ func SystemDataFromContext(ctx context.Context) (*arm.SystemData, error) {
 	if !ok {
 		err := &ContextError{
 			got: systemData,
+			key: contextKeySystemData,
 		}
 		return systemData, err
 	}
 	return systemData, nil
 }
+
+func ContextWithPattern(ctx context.Context, pattern *string) context.Context {
+	return context.WithValue(ctx, contextKeyPattern, pattern)
+}
+
+func PatternFromContext(ctx context.Context) *string {
+	pattern, _ := ctx.Value(contextKeyPattern).(*string)
+	return pattern
+}

frontend/pkg/frontend/frontend.go

@@ -21,6 +21,7 @@ import (
 	azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/Azure/ARO-HCP/frontend/pkg/metrics"
@@ -30,6 +31,8 @@ import (
 	"github.com/Azure/ARO-HCP/internal/ocm"
 )
 
+const healthGaugeName = "frontend_health"
+
 type Frontend struct {
 	clusterServiceClient ocm.ClusterServiceClientSpec
 	listener             net.Listener
@@ -39,16 +42,24 @@ type Frontend struct {
 	dbClient             database.DBClient
 	ready                atomic.Value
 	done                 chan struct{}
-	metrics              Emitter
 	location             string
+	collector            *metrics.SubscriptionCollector
+	healthGauge          prometheus.Gauge
 }
 
-func NewFrontend(logger *slog.Logger, listener net.Listener, metricsListener net.Listener, emitter Emitter, dbClient database.DBClient, location string, csClient ocm.ClusterServiceClientSpec) *Frontend {
+func NewFrontend(
+	logger *slog.Logger,
+	listener net.Listener,
+	metricsListener net.Listener,
+	reg prometheus.Registerer,
+	dbClient database.DBClient,
+	location string,
+	csClient ocm.ClusterServiceClientSpec,
+) *Frontend {
 	f := &Frontend{
 		clusterServiceClient: csClient,
 		listener:             listener,
 		metricsListener:      metricsListener,
-		metrics:              emitter,
 		server: http.Server{
 			ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
 			BaseContext: func(net.Listener) context.Context {
@@ -64,12 +75,19 @@ func NewFrontend(logger *slog.Logger, listener net.Listener, metricsListener net
 				return ContextWithLogger(context.Background(), logger)
 			},
 		},
-		dbClient: dbClient,
-		done:     make(chan struct{}),
-		location: strings.ToLower(location),
+		dbClient:  dbClient,
+		done:      make(chan struct{}),
+		location:  strings.ToLower(location),
+		collector: metrics.NewSubscriptionCollector(reg, dbClient, location),
+		healthGauge: promauto.With(reg).NewGauge(
+			prometheus.GaugeOpts{
+				Name: healthGaugeName,
+				Help: "Reports the health status of the service (0: not healthy, 1: healthy).",
+			},
+		),
 	}
 
-	f.server.Handler = f.routes()
+	f.server.Handler = f.routes(reg)
 	f.metricsServer.Handler = f.metricsRoutes()
 
 	return f
@@ -101,8 +119,7 @@ func (f *Frontend) Run(ctx context.Context, stop <-chan struct{}) {
 		return f.metricsServer.Serve(f.metricsListener)
 	})
 	errs.Go(func() error {
-		collector := metrics.NewSubscriptionCollector(prometheus.DefaultRegisterer, f.dbClient, f.location)
-		collector.Run(logger, stop)
+		f.collector.Run(logger, stop)
 		return nil
 	})
 
@@ -137,19 +154,14 @@ func (f *Frontend) NotFound(writer http.ResponseWriter, request *http.Request) {
 }
 
 func (f *Frontend) Healthz(writer http.ResponseWriter, request *http.Request) {
-	var healthStatus float64
-
 	if f.CheckReady(request.Context()) {
 		writer.WriteHeader(http.StatusOK)
-		healthStatus = 1.0
-	} else {
-		arm.WriteInternalServerError(writer)
-		healthStatus = 0.0
+		f.healthGauge.Set(1.0)
+		return
 	}
 
-	f.metrics.EmitGauge("frontend_health", healthStatus, map[string]string{
-		"endpoint": "/healthz",
-	})
+	arm.WriteInternalServerError(writer)
+	f.healthGauge.Set(0.0)
 }
 
 func (f *Frontend) ArmResourceList(writer http.ResponseWriter, request *http.Request) {