add init duration to prometheus metrics

Signed-off-by: Mike Mason <mason@packet.com>

grpc_server.go

@@ -16,6 +16,7 @@ import (
 	"github.com/packethost/hegel/grpc/hegel"
 	"github.com/packethost/hegel/metrics"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	tink "github.com/tinkerbell/tink/protos/hardware"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/peer"
@@ -213,13 +214,15 @@ func (s *server) Subscribe(in *hegel.SubscribeRequest, stream hegel.Hegel_Subscr
 	startedAt := time.Now().UTC()
 	metrics.TotalSubscriptions.Inc()
 	metrics.Subscriptions.WithLabelValues("initializing").Inc()
+	timer := prometheus.NewTimer(metrics.InitDuration)
 
 	logger := s.log.With("op", "subscribe")
 
 	initError := func(err error) error {
 		logger.Error(err)
 		metrics.Subscriptions.WithLabelValues("initializing").Dec()
 		metrics.Errors.WithLabelValues("subscribe", "initializing").Inc()
+		timer.ObserveDuration()
 		return err
 	}
 
@@ -317,6 +320,7 @@ func (s *server) Subscribe(in *hegel.SubscribeRequest, stream hegel.Hegel_Subscr
 		}
 	}()
 
+	timer.ObserveDuration()
 	metrics.Subscriptions.WithLabelValues("initializing").Dec()
 	metrics.Subscriptions.WithLabelValues("active").Inc()
 

metrics/metrics.go

@@ -15,6 +15,7 @@ const (
 var (
 	CacherConnected    prometheus.Gauge
 	CacherHealthcheck  *prometheus.CounterVec
+	InitDuration       prometheus.Observer
 	Errors             *prometheus.CounterVec
 	MetadataRequests   prometheus.Counter
 	State              prometheus.Gauge
@@ -39,6 +40,12 @@ func Init(_ log.Logger) {
 	}
 	initCounterLabels(CacherHealthcheck, labelValues)
 
+	InitDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "hegel_subscription_initialization_duration_seconds",
+		Help:    "Duration taken to get a responce for a newly discovered request.",
+		Buckets: []float64{0.5, 1, 5, 10, 30, 60},
+	}, []string{}).With(prometheus.Labels{})
+
 	Errors = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "hegel_errors",
 		Help: "Number of errors tracked by hegel",