diff --git a/grpc_server.go b/grpc_server.go index 8d8ec389..c51f09be 100644 --- a/grpc_server.go +++ b/grpc_server.go @@ -16,6 +16,7 @@ import ( "github.com/packethost/hegel/grpc/hegel" "github.com/packethost/hegel/metrics" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" tink "github.com/tinkerbell/tink/protos/hardware" "google.golang.org/grpc/codes" "google.golang.org/grpc/peer" @@ -213,6 +214,7 @@ func (s *server) Subscribe(in *hegel.SubscribeRequest, stream hegel.Hegel_Subscr startedAt := time.Now().UTC() metrics.TotalSubscriptions.Inc() metrics.Subscriptions.WithLabelValues("initializing").Inc() + timer := prometheus.NewTimer(metrics.InitDuration) logger := s.log.With("op", "subscribe") @@ -220,6 +222,7 @@ func (s *server) Subscribe(in *hegel.SubscribeRequest, stream hegel.Hegel_Subscr logger.Error(err) metrics.Subscriptions.WithLabelValues("initializing").Dec() metrics.Errors.WithLabelValues("subscribe", "initializing").Inc() + timer.ObserveDuration() return err } @@ -317,6 +320,7 @@ func (s *server) Subscribe(in *hegel.SubscribeRequest, stream hegel.Hegel_Subscr } }() + timer.ObserveDuration() metrics.Subscriptions.WithLabelValues("initializing").Dec() metrics.Subscriptions.WithLabelValues("active").Inc() diff --git a/metrics/metrics.go b/metrics/metrics.go index b0e3af1c..9db3280b 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -15,6 +15,7 @@ const ( var ( CacherConnected prometheus.Gauge CacherHealthcheck *prometheus.CounterVec + InitDuration prometheus.Observer Errors *prometheus.CounterVec MetadataRequests prometheus.Counter State prometheus.Gauge @@ -39,6 +40,12 @@ func Init(_ log.Logger) { } initCounterLabels(CacherHealthcheck, labelValues) + InitDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "hegel_subscription_initialization_duration_seconds", + Help: "Duration taken to get a responce for a newly discovered request.", + Buckets: []float64{0.5, 1, 5, 10, 30, 60}, + }, []string{}).With(prometheus.Labels{}) + Errors = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "hegel_errors", Help: "Number of errors tracked by hegel",