From c3e4df6de104c98e4bfca768ea4e8fa1bebd5f6b Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 14 Jan 2025 16:58:52 -0500 Subject: [PATCH 1/4] Switch from grpc-ecosystem/go-grpc-prometheus to grpc-ecosystem/go-grpc-middleware/providers/prometheus Signed-off-by: Davanum Srinivas --- bill-of-materials.json | 15 ++- bill-of-materials.override.json | 9 ++ client/v3/go.mod | 3 +- client/v3/go.sum | 6 +- etcdctl/go.sum | 6 +- etcdutl/go.mod | 3 +- etcdutl/go.sum | 6 +- go.mod | 3 +- go.sum | 6 +- server/config/config.go | 3 + server/embed/etcd.go | 6 +- server/etcdmain/grpc_proxy.go | 10 +- server/etcdserver/api/v3rpc/grpc.go | 24 +++- server/etcdserver/api/v3rpc/metrics.go | 52 +++++++- server/go.mod | 3 +- server/go.sum | 6 +- tests/framework/integration/cluster.go | 4 + tests/go.mod | 3 +- tests/go.sum | 6 +- .../clientv3/examples/example_metrics_test.go | 9 +- tests/integration/clientv3/metrics_test.go | 118 ++++++++---------- 21 files changed, 202 insertions(+), 99 deletions(-) diff --git a/bill-of-materials.json b/bill-of-materials.json index b495c09e337..16fd5131fd8 100644 --- a/bill-of-materials.json +++ b/bill-of-materials.json @@ -216,7 +216,16 @@ ] }, { - "project": "github.com/grpc-ecosystem/go-grpc-prometheus", + "project": "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors", "licenses": [ { "type": "Apache License 2.0", @@ -785,6 +794,10 @@ { "project": "sigs.k8s.io/yaml", "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + }, { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 1 diff --git a/bill-of-materials.override.json b/bill-of-materials.override.json index 15afc56409d..9d8d2e9bc3b 100644 --- a/bill-of-materials.override.json +++ b/bill-of-materials.override.json @@ -7,6 +7,15 @@ } ] }, + { + "project": "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, { "project": "github.com/inconshreveable/mousetrap", "licenses": [ diff --git a/client/v3/go.mod b/client/v3/go.mod index 939216ecb90..0aebfba0bd3 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.5 require ( github.com/coreos/go-semver v0.3.1 github.com/dustin/go-humanize v1.0.1 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/prometheus/client_golang v1.20.5 github.com/stretchr/testify v1.10.0 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 @@ -24,6 +24,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/client/v3/go.sum b/client/v3/go.sum index 2b02e26f8bf..82fce51ce70 100644 --- a/client/v3/go.sum +++ b/client/v3/go.sum @@ -24,8 +24,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/etcdctl/go.sum b/etcdctl/go.sum index 7e21083b798..956c83befba 100644 --- a/etcdctl/go.sum +++ b/etcdctl/go.sum @@ -32,8 +32,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/etcdutl/go.mod b/etcdutl/go.mod index d67a673d641..e758d1d08f4 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -52,7 +52,8 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect diff --git a/etcdutl/go.sum b/etcdutl/go.sum index fcb7222aec0..326f5180c6f 100644 --- a/etcdutl/go.sum +++ b/etcdutl/go.sum @@ -37,8 +37,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/go.mod b/go.mod index 026117dc96d..bad0af2a6dd 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,8 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect diff --git a/go.sum b/go.sum index dea496d1f80..77cd78df5c0 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/server/config/config.go b/server/config/config.go index 93bfb3362e3..986bde50e78 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -211,6 +211,9 @@ type ServerConfig struct { // ServerFeatureGate is a server level feature gate ServerFeatureGate featuregate.FeatureGate + + // Metrics types of metrics - should be either 'basic' or 'extensive' + Metrics string } // VerifyBootstrap sanity-checks the initial config for bootstrap case diff --git a/server/embed/etcd.go b/server/embed/etcd.go index bc9725171a6..5b3c3677beb 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -31,7 +31,6 @@ import ( "sync" "time" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/soheilhy/cmux" "go.uber.org/zap" "google.golang.org/grpc" @@ -239,6 +238,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { V2Deprecation: cfg.V2DeprecationEffective(), ExperimentalLocalAddress: cfg.InferLocalAddr(), ServerFeatureGate: cfg.ServerFeatureGate, + Metrics: cfg.Metrics, } if srvcfg.ExperimentalEnableDistributedTracing { @@ -876,10 +876,6 @@ func (e *Etcd) createMetricsListener(murl url.URL) (net.Listener, error) { } func (e *Etcd) serveMetrics() (err error) { - if e.cfg.Metrics == "extensive" { - grpc_prometheus.EnableHandlingTimeHistogram() - } - if len(e.cfg.ListenMetricsUrls) > 0 { metricsMux := http.NewServeMux() etcdhttp.HandleMetrics(metricsMux) diff --git a/server/etcdmain/grpc_proxy.go b/server/etcdmain/grpc_proxy.go index a0df3f99ae4..9aeae162c20 100644 --- a/server/etcdmain/grpc_proxy.go +++ b/server/etcdmain/grpc_proxy.go @@ -30,8 +30,9 @@ import ( "time" grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/prometheus/client_golang/prometheus" "github.com/soheilhy/cmux" "github.com/spf13/cobra" "go.uber.org/zap" @@ -505,11 +506,14 @@ func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server { alwaysLoggingDeciderServer := func(ctx context.Context, fullMethodName string, servingObject any) bool { return true } + serverMetrics := grpc_prometheus.NewServerMetrics() + prometheus.MustRegister(serverMetrics) + grpcChainStreamList := []grpc.StreamServerInterceptor{ - grpc_prometheus.StreamServerInterceptor, + serverMetrics.StreamServerInterceptor(), } grpcChainUnaryList := []grpc.UnaryServerInterceptor{ - grpc_prometheus.UnaryServerInterceptor, + serverMetrics.UnaryServerInterceptor(), } if grpcProxyEnableLogging { grpcChainStreamList = append(grpcChainStreamList, diff --git a/server/etcdserver/api/v3rpc/grpc.go b/server/etcdserver/api/v3rpc/grpc.go index 32949207805..5ef83181606 100644 --- a/server/etcdserver/api/v3rpc/grpc.go +++ b/server/etcdserver/api/v3rpc/grpc.go @@ -18,8 +18,10 @@ import ( "crypto/tls" "math" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" @@ -39,10 +41,17 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer if tls != nil { opts = append(opts, grpc.Creds(credentials.NewTransportCredential(tls))) } + + serverMetrics := grpc_prometheus.NewServerMetrics() + err := prometheus.Register(serverMetrics) + if err != nil { + s.Cfg.Logger.Warn("etcdserver: failed to register grpc metrics: ", zap.Error(err)) + } + chainUnaryInterceptors := []grpc.UnaryServerInterceptor{ newLogUnaryInterceptor(s), newUnaryInterceptor(s), - grpc_prometheus.UnaryServerInterceptor, + serverMetrics.UnaryServerInterceptor(), } if interceptor != nil { chainUnaryInterceptors = append(chainUnaryInterceptors, interceptor) @@ -50,7 +59,14 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer chainStreamInterceptors := []grpc.StreamServerInterceptor{ newStreamInterceptor(s), - grpc_prometheus.StreamServerInterceptor, + serverMetrics.StreamServerInterceptor(), + } + + // If extensive metrics are enabled, register a histogram to track the reponse latency of gRPC requests + if s.Cfg.Metrics == "extensive" { + unaryInterceptor, streamInterceptor := constructExtensiveMetricsInterceptors() + chainUnaryInterceptors = append(chainUnaryInterceptors, unaryInterceptor) + chainStreamInterceptors = append(chainStreamInterceptors, streamInterceptor) } if s.Cfg.ExperimentalEnableDistributedTracing { @@ -79,7 +95,7 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer pb.RegisterMaintenanceServer(grpcServer, NewMaintenanceServer(s, healthNotifier)) // set zero values for metrics registered for this grpc server - grpc_prometheus.Register(grpcServer) + serverMetrics.InitializeMetrics(grpcServer) return grpcServer } diff --git a/server/etcdserver/api/v3rpc/metrics.go b/server/etcdserver/api/v3rpc/metrics.go index 9b432621ec5..0a38976a8bf 100644 --- a/server/etcdserver/api/v3rpc/metrics.go +++ b/server/etcdserver/api/v3rpc/metrics.go @@ -14,7 +14,14 @@ package v3rpc -import "github.com/prometheus/client_golang/prometheus" +import ( + "context" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" +) var ( sentBytes = prometheus.NewCounter(prometheus.CounterOpts{ @@ -58,3 +65,46 @@ func init() { prometheus.MustRegister(streamFailures) prometheus.MustRegister(clientRequests) } + +func splitMethodName(fullMethodName string) (string, string) { + fullMethodName = strings.TrimPrefix(fullMethodName, "/") // remove leading slash + if i := strings.Index(fullMethodName, "/"); i >= 0 { + return fullMethodName[:i], fullMethodName[i+1:] + } + return "unknown", "unknown" +} + +// constructExtensiveMetricsInterceptors constructs unary and stream interceptors to record histogram metrics for gRPC requests +func constructExtensiveMetricsInterceptors() (grpc.UnaryServerInterceptor, grpc.StreamServerInterceptor) { + // Define a new histogram metric using default buckets + serverHandledHistogram := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "grpc_server_handling_seconds", + Help: "Histogram of response latency (seconds) of gRPC that had been application-level handled by the server.", + Buckets: prometheus.DefBuckets, + }, + []string{"grpc_type", "grpc_service", "grpc_method"}, + ) + prometheus.MustRegister(serverHandledHistogram) + + // method to record histogram metrics for both unary and stream requests + recordHistogramMetrics := func(serverHandledHistogram *prometheus.HistogramVec, grpcType, fullMethodName string, startTime time.Time) { + grpcService, grpcMethod := splitMethodName(fullMethodName) + serverHandledHistogram.WithLabelValues(grpcType, grpcService, grpcMethod).Observe(time.Since(startTime).Seconds()) + } + + // Add a new interceptor to spit out histogram metrics for unary requests + unaryInterceptor := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + startTime := time.Now() + resp, err = handler(ctx, req) + recordHistogramMetrics(serverHandledHistogram, "unary", info.FullMethod, startTime) + return resp, err + } + streamInterceptor := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + startTime := time.Now() + err := handler(srv, ss) + recordHistogramMetrics(serverHandledHistogram, "stream", info.FullMethod, startTime) + return err + } + return unaryInterceptor, streamInterceptor +} diff --git a/server/go.mod b/server/go.mod index e7e8c5a266e..40556abb845 100644 --- a/server/go.mod +++ b/server/go.mod @@ -15,7 +15,7 @@ require ( github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 github.com/jonboulle/clockwork v0.5.0 github.com/prometheus/client_golang v1.20.5 @@ -56,6 +56,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/server/go.sum b/server/go.sum index 8b0fb813e8b..ed7ac3830d2 100644 --- a/server/go.sum +++ b/server/go.sum @@ -59,8 +59,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/tests/framework/integration/cluster.go b/tests/framework/integration/cluster.go index 10c718764b9..c81c10d0038 100644 --- a/tests/framework/integration/cluster.go +++ b/tests/framework/integration/cluster.go @@ -175,6 +175,7 @@ type ClusterConfig struct { MaxLearners int DisableStrictReconfigCheck bool CorruptCheckTime time.Duration + Metrics string } type Cluster struct { @@ -292,6 +293,7 @@ func (c *Cluster) MustNewMember(t testutil.TB) *Member { MaxLearners: c.Cfg.MaxLearners, DisableStrictReconfigCheck: c.Cfg.DisableStrictReconfigCheck, CorruptCheckTime: c.Cfg.CorruptCheckTime, + Metrics: c.Cfg.Metrics, }) m.DiscoveryURL = c.Cfg.DiscoveryURL return m @@ -617,6 +619,7 @@ type MemberConfig struct { MaxLearners int DisableStrictReconfigCheck bool CorruptCheckTime time.Duration + Metrics string } // MustNewMember return an inited member with the given name. If peerTLS is @@ -731,6 +734,7 @@ func MustNewMember(t testutil.TB, mcfg MemberConfig) *Member { if mcfg.MaxLearners != 0 { m.MaxLearners = mcfg.MaxLearners } + m.Metrics = mcfg.Metrics m.V2Deprecation = config.V2_DEPR_DEFAULT m.GRPCServerRecorder = &grpctesting.GRPCRecorder{} diff --git a/tests/go.mod b/tests/go.mod index 2daab290525..492c177af0d 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -21,7 +21,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/go-cmp v0.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 @@ -71,6 +71,7 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/klauspost/compress v1.17.9 // indirect diff --git a/tests/go.sum b/tests/go.sum index 8cfcd1df8a7..8f13c0c4d0b 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -71,8 +71,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/tests/integration/clientv3/examples/example_metrics_test.go b/tests/integration/clientv3/examples/example_metrics_test.go index d21c6d393e2..ebc71c9de45 100644 --- a/tests/integration/clientv3/examples/example_metrics_test.go +++ b/tests/integration/clientv3/examples/example_metrics_test.go @@ -23,7 +23,8 @@ import ( "net/http" "strings" - grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" + grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" @@ -35,11 +36,13 @@ func mockClient_metrics() { func ExampleClient_metrics() { forUnitTestsRunInMockedContext(mockClient_metrics, func() { + clientMetrics := grpcprom.NewClientMetrics() + prometheus.Register(clientMetrics) cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialOptions: []grpc.DialOption{ - grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor), - grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor), + grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), + grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), }, }) if err != nil { diff --git a/tests/integration/clientv3/metrics_test.go b/tests/integration/clientv3/metrics_test.go index 252ad6ec815..92959e3d121 100644 --- a/tests/integration/clientv3/metrics_test.go +++ b/tests/integration/clientv3/metrics_test.go @@ -16,6 +16,7 @@ package clientv3test import ( "bufio" + "bytes" "context" "errors" "io" @@ -27,9 +28,10 @@ import ( "testing" "time" - grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" + grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/expfmt" "google.golang.org/grpc" "go.etcd.io/etcd/client/pkg/v3/transport" @@ -75,11 +77,14 @@ func TestV3ClientMetrics(t *testing.T) { clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1}) defer clus.Terminate(t) + clientMetrics := grpcprom.NewClientMetrics() + prometheus.Register(clientMetrics) + cfg := clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialOptions: []grpc.DialOption{ - grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor), - grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor), + grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), + grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), }, } cli, cerr := integration2.NewClient(t, cfg) @@ -147,6 +152,24 @@ func sumCountersForMetricAndLabels(t *testing.T, url string, metricName string, } func getHTTPBodyAsLines(t *testing.T, url string) []string { + data := getHTTPBodyAsBytes(t, url) + + reader := bufio.NewReader(bytes.NewReader(data)) + var lines []string + for { + line, err := reader.ReadString('\n') + if err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("error reading: %v", err) + } + lines = append(lines, line) + } + return lines +} + +func getHTTPBodyAsBytes(t *testing.T, url string) []byte { cfgtls := transport.TLSInfo{} tr, err := transport.NewTransport(cfgtls, time.Second) if err != nil { @@ -162,21 +185,12 @@ func getHTTPBodyAsLines(t *testing.T, url string) []string { if err != nil { t.Fatalf("Error fetching: %v", err) } - - reader := bufio.NewReader(resp.Body) - var lines []string - for { - line, err := reader.ReadString('\n') - if err != nil { - if errors.Is(err, io.EOF) { - break - } - t.Fatalf("error reading: %v", err) - } - lines = append(lines, line) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Error reading http body: %v", err) } - resp.Body.Close() - return lines + return body } func TestAllMetricsGenerated(t *testing.T) { @@ -213,7 +227,7 @@ func TestAllMetricsGenerated(t *testing.T) { url := "unix://" + addr + "/metrics" - clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1}) + clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1, Metrics: "extensive"}) defer clus.Terminate(t) clientMetrics := grpcprom.NewClientMetrics() @@ -249,42 +263,26 @@ func TestAllMetricsGenerated(t *testing.T) { // Define the expected list of metrics expectedMetrics := []string{ "etcd_cluster_version", - "etcd_disk_backend_commit_duration_seconds_bucket", - "etcd_disk_backend_commit_duration_seconds_count", - "etcd_disk_backend_commit_duration_seconds_sum", - "etcd_disk_backend_defrag_duration_seconds_bucket", - "etcd_disk_backend_defrag_duration_seconds_count", - "etcd_disk_backend_defrag_duration_seconds_sum", - "etcd_disk_backend_snapshot_duration_seconds_bucket", - "etcd_disk_backend_snapshot_duration_seconds_count", - "etcd_disk_backend_snapshot_duration_seconds_sum", + "etcd_disk_backend_commit_duration_seconds", + "etcd_disk_backend_defrag_duration_seconds", + "etcd_disk_backend_snapshot_duration_seconds", "etcd_disk_defrag_inflight", - "etcd_disk_wal_fsync_duration_seconds_bucket", - "etcd_disk_wal_fsync_duration_seconds_count", - "etcd_disk_wal_fsync_duration_seconds_sum", + "etcd_disk_wal_fsync_duration_seconds", "etcd_disk_wal_write_bytes_total", - "etcd_disk_wal_write_duration_seconds_bucket", - "etcd_disk_wal_write_duration_seconds_count", - "etcd_disk_wal_write_duration_seconds_sum", + "etcd_disk_wal_write_duration_seconds", "etcd_mvcc_db_open_read_transactions", "etcd_mvcc_db_total_size_in_bytes", "etcd_mvcc_db_total_size_in_use_in_bytes", "etcd_mvcc_delete_total", - "etcd_mvcc_hash_duration_seconds_bucket", - "etcd_mvcc_hash_duration_seconds_count", - "etcd_mvcc_hash_duration_seconds_sum", - "etcd_mvcc_hash_rev_duration_seconds_bucket", - "etcd_mvcc_hash_rev_duration_seconds_count", - "etcd_mvcc_hash_rev_duration_seconds_sum", + "etcd_mvcc_hash_duration_seconds", + "etcd_mvcc_hash_rev_duration_seconds", "etcd_mvcc_put_total", "etcd_mvcc_range_total", "etcd_mvcc_txn_total", "etcd_network_client_grpc_received_bytes_total", "etcd_network_client_grpc_sent_bytes_total", "etcd_network_known_peers", - "etcd_server_apply_duration_seconds_bucket", - "etcd_server_apply_duration_seconds_count", - "etcd_server_apply_duration_seconds_sum", + "etcd_server_apply_duration_seconds", "etcd_server_client_requests_total", "etcd_server_go_version", "etcd_server_has_leader", @@ -306,20 +304,15 @@ func TestAllMetricsGenerated(t *testing.T) { "etcd_server_slow_read_indexes_total", "etcd_server_snapshot_apply_in_progress_total", "etcd_server_version", - "etcd_snap_db_fsync_duration_seconds_bucket", - "etcd_snap_db_fsync_duration_seconds_count", - "etcd_snap_db_fsync_duration_seconds_sum", - "etcd_snap_db_save_total_duration_seconds_bucket", - "etcd_snap_db_save_total_duration_seconds_count", - "etcd_snap_db_save_total_duration_seconds_sum", - "etcd_snap_fsync_duration_seconds_bucket", - "etcd_snap_fsync_duration_seconds_count", - "etcd_snap_fsync_duration_seconds_sum", + "etcd_snap_db_fsync_duration_seconds", + "etcd_snap_db_save_total_duration_seconds", + "etcd_snap_fsync_duration_seconds", "grpc_client_handled_total", "grpc_client_msg_received_total", "grpc_client_msg_sent_total", "grpc_client_started_total", "grpc_server_handled_total", + "grpc_server_handling_seconds", "grpc_server_msg_received_total", "grpc_server_msg_sent_total", "grpc_server_started_total", @@ -335,20 +328,15 @@ func TestAllMetricsGenerated(t *testing.T) { } func getMetricsList(t *testing.T, url string) []string { - lines := getHTTPBodyAsLines(t, url) - metrics := make(map[string]struct{}) - for _, line := range lines { - if strings.Contains(line, "{") { - metric := line[:strings.Index(line, "{")] - metrics[metric] = struct{}{} - } else { - metric := line[:strings.Index(line, " ")] - metrics[metric] = struct{}{} - } + data := getHTTPBodyAsBytes(t, url) + var parser expfmt.TextParser + mfs, err := parser.TextToMetricFamilies(bytes.NewReader(data)) + if err != nil { + t.Errorf("Failed to parse metric families") } - var metricList []string - for metric := range metrics { - metricList = append(metricList, metric) + var ms []string + for key := range mfs { + ms = append(ms, key) } - return metricList + return ms } From 3648220e22d98ac2c7614511158ce6b15a27a08a Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Fri, 31 Jan 2025 06:44:29 -0500 Subject: [PATCH 2/4] incorporate feedback from review Signed-off-by: Davanum Srinivas --- server/etcdserver/api/v3rpc/grpc.go | 15 +++---- server/etcdserver/api/v3rpc/metrics.go | 48 ---------------------- tests/integration/clientv3/metrics_test.go | 7 +--- 3 files changed, 8 insertions(+), 62 deletions(-) diff --git a/server/etcdserver/api/v3rpc/grpc.go b/server/etcdserver/api/v3rpc/grpc.go index 5ef83181606..810fd219b8f 100644 --- a/server/etcdserver/api/v3rpc/grpc.go +++ b/server/etcdserver/api/v3rpc/grpc.go @@ -42,10 +42,14 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer opts = append(opts, grpc.Creds(credentials.NewTransportCredential(tls))) } - serverMetrics := grpc_prometheus.NewServerMetrics() + var mopts []grpc_prometheus.ServerMetricsOption + if s.Cfg.Metrics == "extensive" { + mopts = append(mopts, grpc_prometheus.WithServerHandlingTimeHistogram()) + } + serverMetrics := grpc_prometheus.NewServerMetrics(mopts...) err := prometheus.Register(serverMetrics) if err != nil { - s.Cfg.Logger.Warn("etcdserver: failed to register grpc metrics: ", zap.Error(err)) + s.Cfg.Logger.Warn("etcdserver: failed to register grpc metrics", zap.Error(err)) } chainUnaryInterceptors := []grpc.UnaryServerInterceptor{ @@ -62,13 +66,6 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer serverMetrics.StreamServerInterceptor(), } - // If extensive metrics are enabled, register a histogram to track the reponse latency of gRPC requests - if s.Cfg.Metrics == "extensive" { - unaryInterceptor, streamInterceptor := constructExtensiveMetricsInterceptors() - chainUnaryInterceptors = append(chainUnaryInterceptors, unaryInterceptor) - chainStreamInterceptors = append(chainStreamInterceptors, streamInterceptor) - } - if s.Cfg.ExperimentalEnableDistributedTracing { chainUnaryInterceptors = append(chainUnaryInterceptors, otelgrpc.UnaryServerInterceptor(s.Cfg.ExperimentalTracerOptions...)) chainStreamInterceptors = append(chainStreamInterceptors, otelgrpc.StreamServerInterceptor(s.Cfg.ExperimentalTracerOptions...)) diff --git a/server/etcdserver/api/v3rpc/metrics.go b/server/etcdserver/api/v3rpc/metrics.go index 0a38976a8bf..d79506e0732 100644 --- a/server/etcdserver/api/v3rpc/metrics.go +++ b/server/etcdserver/api/v3rpc/metrics.go @@ -15,12 +15,7 @@ package v3rpc import ( - "context" - "strings" - "time" - "github.com/prometheus/client_golang/prometheus" - "google.golang.org/grpc" ) var ( @@ -65,46 +60,3 @@ func init() { prometheus.MustRegister(streamFailures) prometheus.MustRegister(clientRequests) } - -func splitMethodName(fullMethodName string) (string, string) { - fullMethodName = strings.TrimPrefix(fullMethodName, "/") // remove leading slash - if i := strings.Index(fullMethodName, "/"); i >= 0 { - return fullMethodName[:i], fullMethodName[i+1:] - } - return "unknown", "unknown" -} - -// constructExtensiveMetricsInterceptors constructs unary and stream interceptors to record histogram metrics for gRPC requests -func constructExtensiveMetricsInterceptors() (grpc.UnaryServerInterceptor, grpc.StreamServerInterceptor) { - // Define a new histogram metric using default buckets - serverHandledHistogram := prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "grpc_server_handling_seconds", - Help: "Histogram of response latency (seconds) of gRPC that had been application-level handled by the server.", - Buckets: prometheus.DefBuckets, - }, - []string{"grpc_type", "grpc_service", "grpc_method"}, - ) - prometheus.MustRegister(serverHandledHistogram) - - // method to record histogram metrics for both unary and stream requests - recordHistogramMetrics := func(serverHandledHistogram *prometheus.HistogramVec, grpcType, fullMethodName string, startTime time.Time) { - grpcService, grpcMethod := splitMethodName(fullMethodName) - serverHandledHistogram.WithLabelValues(grpcType, grpcService, grpcMethod).Observe(time.Since(startTime).Seconds()) - } - - // Add a new interceptor to spit out histogram metrics for unary requests - unaryInterceptor := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { - startTime := time.Now() - resp, err = handler(ctx, req) - recordHistogramMetrics(serverHandledHistogram, "unary", info.FullMethod, startTime) - return resp, err - } - streamInterceptor := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - startTime := time.Now() - err := handler(srv, ss) - recordHistogramMetrics(serverHandledHistogram, "stream", info.FullMethod, startTime) - return err - } - return unaryInterceptor, streamInterceptor -} diff --git a/tests/integration/clientv3/metrics_test.go b/tests/integration/clientv3/metrics_test.go index 92959e3d121..8a76a3caad4 100644 --- a/tests/integration/clientv3/metrics_test.go +++ b/tests/integration/clientv3/metrics_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "io" + "maps" "net" "net/http" "slices" @@ -334,9 +335,5 @@ func getMetricsList(t *testing.T, url string) []string { if err != nil { t.Errorf("Failed to parse metric families") } - var ms []string - for key := range mfs { - ms = append(ms, key) - } - return ms + return slices.Collect(maps.Keys(mfs)) } From 1f33bafd015ed5c5fa95ac8e0f38057cb8c079ef Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Fri, 31 Jan 2025 14:31:19 +0000 Subject: [PATCH 3/4] Remove the integration test case TestAllMetricsGenerated Refer to https://github.com/etcd-io/etcd/pull/19195#issuecomment-2627486682 Signed-off-by: Benjamin Wang --- tests/integration/clientv3/metrics_test.go | 147 --------------------- 1 file changed, 147 deletions(-) diff --git a/tests/integration/clientv3/metrics_test.go b/tests/integration/clientv3/metrics_test.go index 8a76a3caad4..3188276c9c2 100644 --- a/tests/integration/clientv3/metrics_test.go +++ b/tests/integration/clientv3/metrics_test.go @@ -20,10 +20,8 @@ import ( "context" "errors" "io" - "maps" "net" "net/http" - "slices" "strconv" "strings" "testing" @@ -32,7 +30,6 @@ import ( grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/common/expfmt" "google.golang.org/grpc" "go.etcd.io/etcd/client/pkg/v3/transport" @@ -193,147 +190,3 @@ func getHTTPBodyAsBytes(t *testing.T, url string) []byte { } return body } - -func TestAllMetricsGenerated(t *testing.T) { - integration2.BeforeTest(t) - - var ( - addr = "localhost:27989" - ln net.Listener - ) - - srv := &http.Server{Handler: promhttp.Handler()} - srv.SetKeepAlivesEnabled(false) - - ln, err := transport.NewUnixListener(addr) - if err != nil { - t.Errorf("Error: %v occurred while listening on addr: %v", err, addr) - } - - donec := make(chan struct{}) - defer func() { - ln.Close() - <-donec - }() - - // listen for all Prometheus metrics - go func() { - defer close(donec) - - serr := srv.Serve(ln) - if serr != nil && !transport.IsClosedConnError(serr) { - t.Errorf("Err serving http requests: %v", serr) - } - }() - - url := "unix://" + addr + "/metrics" - - clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1, Metrics: "extensive"}) - defer clus.Terminate(t) - - clientMetrics := grpcprom.NewClientMetrics() - prometheus.Register(clientMetrics) - - cfg := clientv3.Config{ - Endpoints: []string{clus.Members[0].GRPCURL}, - DialOptions: []grpc.DialOption{ - grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), - grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), - }, - } - cli, cerr := integration2.NewClient(t, cfg) - if cerr != nil { - t.Fatal(cerr) - } - defer cli.Close() - - // Perform some operations to generate metrics - wc := cli.Watch(context.Background(), "foo") - _, err = cli.Put(context.Background(), "foo", "bar") - if err != nil { - t.Errorf("Error putting value in key store") - } - - // consume watch response - select { - case <-wc: - case <-time.After(10 * time.Second): - t.Error("Timeout occurred for getting watch response") - } - - // Define the expected list of metrics - expectedMetrics := []string{ - "etcd_cluster_version", - "etcd_disk_backend_commit_duration_seconds", - "etcd_disk_backend_defrag_duration_seconds", - "etcd_disk_backend_snapshot_duration_seconds", - "etcd_disk_defrag_inflight", - "etcd_disk_wal_fsync_duration_seconds", - "etcd_disk_wal_write_bytes_total", - "etcd_disk_wal_write_duration_seconds", - "etcd_mvcc_db_open_read_transactions", - "etcd_mvcc_db_total_size_in_bytes", - "etcd_mvcc_db_total_size_in_use_in_bytes", - "etcd_mvcc_delete_total", - "etcd_mvcc_hash_duration_seconds", - "etcd_mvcc_hash_rev_duration_seconds", - "etcd_mvcc_put_total", - "etcd_mvcc_range_total", - "etcd_mvcc_txn_total", - "etcd_network_client_grpc_received_bytes_total", - "etcd_network_client_grpc_sent_bytes_total", - "etcd_network_known_peers", - "etcd_server_apply_duration_seconds", - "etcd_server_client_requests_total", - "etcd_server_go_version", - "etcd_server_has_leader", - "etcd_server_health_failures", - "etcd_server_health_success", - "etcd_server_heartbeat_send_failures_total", - "etcd_server_id", - "etcd_server_is_leader", - "etcd_server_is_learner", - "etcd_server_leader_changes_seen_total", - "etcd_server_learner_promote_successes", - "etcd_server_proposals_applied_total", - "etcd_server_proposals_committed_total", - "etcd_server_proposals_failed_total", - "etcd_server_proposals_pending", - "etcd_server_quota_backend_bytes", - "etcd_server_read_indexes_failed_total", - "etcd_server_slow_apply_total", - "etcd_server_slow_read_indexes_total", - "etcd_server_snapshot_apply_in_progress_total", - "etcd_server_version", - "etcd_snap_db_fsync_duration_seconds", - "etcd_snap_db_save_total_duration_seconds", - "etcd_snap_fsync_duration_seconds", - "grpc_client_handled_total", - "grpc_client_msg_received_total", - "grpc_client_msg_sent_total", - "grpc_client_started_total", - "grpc_server_handled_total", - "grpc_server_handling_seconds", - "grpc_server_msg_received_total", - "grpc_server_msg_sent_total", - "grpc_server_started_total", - } - - // Get the list of generated metrics - generatedMetrics := getMetricsList(t, url) - for _, metric := range expectedMetrics { - if !slices.Contains(generatedMetrics, metric) { - t.Errorf("Expected metric %s not found in generated metrics", metric) - } - } -} - -func getMetricsList(t *testing.T, url string) []string { - data := getHTTPBodyAsBytes(t, url) - var parser expfmt.TextParser - mfs, err := parser.TextToMetricFamilies(bytes.NewReader(data)) - if err != nil { - t.Errorf("Failed to parse metric families") - } - return slices.Collect(maps.Keys(mfs)) -} From 6d8c6af06abb1b0cb35a27a11419842b95820f66 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Fri, 31 Jan 2025 14:40:32 +0000 Subject: [PATCH 4/4] Update the import items Signed-off-by: Benjamin Wang --- tests/integration/clientv3/examples/example_metrics_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/clientv3/examples/example_metrics_test.go b/tests/integration/clientv3/examples/example_metrics_test.go index ebc71c9de45..75b47e53d23 100644 --- a/tests/integration/clientv3/examples/example_metrics_test.go +++ b/tests/integration/clientv3/examples/example_metrics_test.go @@ -26,8 +26,9 @@ import ( grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" + + clientv3 "go.etcd.io/etcd/client/v3" ) func mockClient_metrics() {