From c3e4df6de104c98e4bfca768ea4e8fa1bebd5f6b Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 14 Jan 2025 16:58:52 -0500 Subject: [PATCH] Switch from grpc-ecosystem/go-grpc-prometheus to grpc-ecosystem/go-grpc-middleware/providers/prometheus Signed-off-by: Davanum Srinivas --- bill-of-materials.json | 15 ++- bill-of-materials.override.json | 9 ++ client/v3/go.mod | 3 +- client/v3/go.sum | 6 +- etcdctl/go.sum | 6 +- etcdutl/go.mod | 3 +- etcdutl/go.sum | 6 +- go.mod | 3 +- go.sum | 6 +- server/config/config.go | 3 + server/embed/etcd.go | 6 +- server/etcdmain/grpc_proxy.go | 10 +- server/etcdserver/api/v3rpc/grpc.go | 24 +++- server/etcdserver/api/v3rpc/metrics.go | 52 +++++++- server/go.mod | 3 +- server/go.sum | 6 +- tests/framework/integration/cluster.go | 4 + tests/go.mod | 3 +- tests/go.sum | 6 +- .../clientv3/examples/example_metrics_test.go | 9 +- tests/integration/clientv3/metrics_test.go | 118 ++++++++---------- 21 files changed, 202 insertions(+), 99 deletions(-) diff --git a/bill-of-materials.json b/bill-of-materials.json index b495c09e3..16fd5131f 100644 --- a/bill-of-materials.json +++ b/bill-of-materials.json @@ -216,7 +216,16 @@ ] }, { - "project": "github.com/grpc-ecosystem/go-grpc-prometheus", + "project": "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors", "licenses": [ { "type": "Apache License 2.0", @@ -785,6 +794,10 @@ { "project": "sigs.k8s.io/yaml", "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + }, { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 1 diff --git a/bill-of-materials.override.json b/bill-of-materials.override.json index 15afc5640..9d8d2e9bc 100644 --- a/bill-of-materials.override.json +++ b/bill-of-materials.override.json @@ -7,6 +7,15 @@ } ] }, + { + "project": "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, { "project": "github.com/inconshreveable/mousetrap", "licenses": [ diff --git a/client/v3/go.mod b/client/v3/go.mod index 939216ecb..0aebfba0b 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.5 require ( github.com/coreos/go-semver v0.3.1 github.com/dustin/go-humanize v1.0.1 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/prometheus/client_golang v1.20.5 github.com/stretchr/testify v1.10.0 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 @@ -24,6 +24,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/client/v3/go.sum b/client/v3/go.sum index 2b02e26f8..82fce51ce 100644 --- a/client/v3/go.sum +++ b/client/v3/go.sum @@ -24,8 +24,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/etcdctl/go.sum b/etcdctl/go.sum index 7e21083b7..956c83bef 100644 --- a/etcdctl/go.sum +++ b/etcdctl/go.sum @@ -32,8 +32,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/etcdutl/go.mod b/etcdutl/go.mod index d67a673d6..e758d1d08 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -52,7 +52,8 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect diff --git a/etcdutl/go.sum b/etcdutl/go.sum index fcb7222ae..326f5180c 100644 --- a/etcdutl/go.sum +++ b/etcdutl/go.sum @@ -37,8 +37,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/go.mod b/go.mod index 026117dc9..bad0af2a6 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,8 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect diff --git a/go.sum b/go.sum index dea496d1f..77cd78df5 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/server/config/config.go b/server/config/config.go index 93bfb3362..986bde50e 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -211,6 +211,9 @@ type ServerConfig struct { // ServerFeatureGate is a server level feature gate ServerFeatureGate featuregate.FeatureGate + + // Metrics types of metrics - should be either 'basic' or 'extensive' + Metrics string } // VerifyBootstrap sanity-checks the initial config for bootstrap case diff --git a/server/embed/etcd.go b/server/embed/etcd.go index bc9725171..5b3c3677b 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -31,7 +31,6 @@ import ( "sync" "time" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/soheilhy/cmux" "go.uber.org/zap" "google.golang.org/grpc" @@ -239,6 +238,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { V2Deprecation: cfg.V2DeprecationEffective(), ExperimentalLocalAddress: cfg.InferLocalAddr(), ServerFeatureGate: cfg.ServerFeatureGate, + Metrics: cfg.Metrics, } if srvcfg.ExperimentalEnableDistributedTracing { @@ -876,10 +876,6 @@ func (e *Etcd) createMetricsListener(murl url.URL) (net.Listener, error) { } func (e *Etcd) serveMetrics() (err error) { - if e.cfg.Metrics == "extensive" { - grpc_prometheus.EnableHandlingTimeHistogram() - } - if len(e.cfg.ListenMetricsUrls) > 0 { metricsMux := http.NewServeMux() etcdhttp.HandleMetrics(metricsMux) diff --git a/server/etcdmain/grpc_proxy.go b/server/etcdmain/grpc_proxy.go index a0df3f99a..9aeae162c 100644 --- a/server/etcdmain/grpc_proxy.go +++ b/server/etcdmain/grpc_proxy.go @@ -30,8 +30,9 @@ import ( "time" grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/prometheus/client_golang/prometheus" "github.com/soheilhy/cmux" "github.com/spf13/cobra" "go.uber.org/zap" @@ -505,11 +506,14 @@ func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server { alwaysLoggingDeciderServer := func(ctx context.Context, fullMethodName string, servingObject any) bool { return true } + serverMetrics := grpc_prometheus.NewServerMetrics() + prometheus.MustRegister(serverMetrics) + grpcChainStreamList := []grpc.StreamServerInterceptor{ - grpc_prometheus.StreamServerInterceptor, + serverMetrics.StreamServerInterceptor(), } grpcChainUnaryList := []grpc.UnaryServerInterceptor{ - grpc_prometheus.UnaryServerInterceptor, + serverMetrics.UnaryServerInterceptor(), } if grpcProxyEnableLogging { grpcChainStreamList = append(grpcChainStreamList, diff --git a/server/etcdserver/api/v3rpc/grpc.go b/server/etcdserver/api/v3rpc/grpc.go index 329492078..5ef831816 100644 --- a/server/etcdserver/api/v3rpc/grpc.go +++ b/server/etcdserver/api/v3rpc/grpc.go @@ -18,8 +18,10 @@ import ( "crypto/tls" "math" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" @@ -39,10 +41,17 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer if tls != nil { opts = append(opts, grpc.Creds(credentials.NewTransportCredential(tls))) } + + serverMetrics := grpc_prometheus.NewServerMetrics() + err := prometheus.Register(serverMetrics) + if err != nil { + s.Cfg.Logger.Warn("etcdserver: failed to register grpc metrics: ", zap.Error(err)) + } + chainUnaryInterceptors := []grpc.UnaryServerInterceptor{ newLogUnaryInterceptor(s), newUnaryInterceptor(s), - grpc_prometheus.UnaryServerInterceptor, + serverMetrics.UnaryServerInterceptor(), } if interceptor != nil { chainUnaryInterceptors = append(chainUnaryInterceptors, interceptor) @@ -50,7 +59,14 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer chainStreamInterceptors := []grpc.StreamServerInterceptor{ newStreamInterceptor(s), - grpc_prometheus.StreamServerInterceptor, + serverMetrics.StreamServerInterceptor(), + } + + // If extensive metrics are enabled, register a histogram to track the reponse latency of gRPC requests + if s.Cfg.Metrics == "extensive" { + unaryInterceptor, streamInterceptor := constructExtensiveMetricsInterceptors() + chainUnaryInterceptors = append(chainUnaryInterceptors, unaryInterceptor) + chainStreamInterceptors = append(chainStreamInterceptors, streamInterceptor) } if s.Cfg.ExperimentalEnableDistributedTracing { @@ -79,7 +95,7 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer pb.RegisterMaintenanceServer(grpcServer, NewMaintenanceServer(s, healthNotifier)) // set zero values for metrics registered for this grpc server - grpc_prometheus.Register(grpcServer) + serverMetrics.InitializeMetrics(grpcServer) return grpcServer } diff --git a/server/etcdserver/api/v3rpc/metrics.go b/server/etcdserver/api/v3rpc/metrics.go index 9b432621e..0a38976a8 100644 --- a/server/etcdserver/api/v3rpc/metrics.go +++ b/server/etcdserver/api/v3rpc/metrics.go @@ -14,7 +14,14 @@ package v3rpc -import "github.com/prometheus/client_golang/prometheus" +import ( + "context" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" +) var ( sentBytes = prometheus.NewCounter(prometheus.CounterOpts{ @@ -58,3 +65,46 @@ func init() { prometheus.MustRegister(streamFailures) prometheus.MustRegister(clientRequests) } + +func splitMethodName(fullMethodName string) (string, string) { + fullMethodName = strings.TrimPrefix(fullMethodName, "/") // remove leading slash + if i := strings.Index(fullMethodName, "/"); i >= 0 { + return fullMethodName[:i], fullMethodName[i+1:] + } + return "unknown", "unknown" +} + +// constructExtensiveMetricsInterceptors constructs unary and stream interceptors to record histogram metrics for gRPC requests +func constructExtensiveMetricsInterceptors() (grpc.UnaryServerInterceptor, grpc.StreamServerInterceptor) { + // Define a new histogram metric using default buckets + serverHandledHistogram := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "grpc_server_handling_seconds", + Help: "Histogram of response latency (seconds) of gRPC that had been application-level handled by the server.", + Buckets: prometheus.DefBuckets, + }, + []string{"grpc_type", "grpc_service", "grpc_method"}, + ) + prometheus.MustRegister(serverHandledHistogram) + + // method to record histogram metrics for both unary and stream requests + recordHistogramMetrics := func(serverHandledHistogram *prometheus.HistogramVec, grpcType, fullMethodName string, startTime time.Time) { + grpcService, grpcMethod := splitMethodName(fullMethodName) + serverHandledHistogram.WithLabelValues(grpcType, grpcService, grpcMethod).Observe(time.Since(startTime).Seconds()) + } + + // Add a new interceptor to spit out histogram metrics for unary requests + unaryInterceptor := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + startTime := time.Now() + resp, err = handler(ctx, req) + recordHistogramMetrics(serverHandledHistogram, "unary", info.FullMethod, startTime) + return resp, err + } + streamInterceptor := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + startTime := time.Now() + err := handler(srv, ss) + recordHistogramMetrics(serverHandledHistogram, "stream", info.FullMethod, startTime) + return err + } + return unaryInterceptor, streamInterceptor +} diff --git a/server/go.mod b/server/go.mod index e7e8c5a26..40556abb8 100644 --- a/server/go.mod +++ b/server/go.mod @@ -15,7 +15,7 @@ require ( github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 github.com/jonboulle/clockwork v0.5.0 github.com/prometheus/client_golang v1.20.5 @@ -56,6 +56,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/server/go.sum b/server/go.sum index 8b0fb813e..ed7ac3830 100644 --- a/server/go.sum +++ b/server/go.sum @@ -59,8 +59,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/tests/framework/integration/cluster.go b/tests/framework/integration/cluster.go index 10c718764..c81c10d00 100644 --- a/tests/framework/integration/cluster.go +++ b/tests/framework/integration/cluster.go @@ -175,6 +175,7 @@ type ClusterConfig struct { MaxLearners int DisableStrictReconfigCheck bool CorruptCheckTime time.Duration + Metrics string } type Cluster struct { @@ -292,6 +293,7 @@ func (c *Cluster) MustNewMember(t testutil.TB) *Member { MaxLearners: c.Cfg.MaxLearners, DisableStrictReconfigCheck: c.Cfg.DisableStrictReconfigCheck, CorruptCheckTime: c.Cfg.CorruptCheckTime, + Metrics: c.Cfg.Metrics, }) m.DiscoveryURL = c.Cfg.DiscoveryURL return m @@ -617,6 +619,7 @@ type MemberConfig struct { MaxLearners int DisableStrictReconfigCheck bool CorruptCheckTime time.Duration + Metrics string } // MustNewMember return an inited member with the given name. If peerTLS is @@ -731,6 +734,7 @@ func MustNewMember(t testutil.TB, mcfg MemberConfig) *Member { if mcfg.MaxLearners != 0 { m.MaxLearners = mcfg.MaxLearners } + m.Metrics = mcfg.Metrics m.V2Deprecation = config.V2_DEPR_DEFAULT m.GRPCServerRecorder = &grpctesting.GRPCRecorder{} diff --git a/tests/go.mod b/tests/go.mod index 2daab2905..492c177af 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -21,7 +21,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/go-cmp v0.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 @@ -71,6 +71,7 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/klauspost/compress v1.17.9 // indirect diff --git a/tests/go.sum b/tests/go.sum index 8cfcd1df8..8f13c0c4d 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -71,8 +71,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/tests/integration/clientv3/examples/example_metrics_test.go b/tests/integration/clientv3/examples/example_metrics_test.go index d21c6d393..ebc71c9de 100644 --- a/tests/integration/clientv3/examples/example_metrics_test.go +++ b/tests/integration/clientv3/examples/example_metrics_test.go @@ -23,7 +23,8 @@ import ( "net/http" "strings" - grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" + grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" @@ -35,11 +36,13 @@ func mockClient_metrics() { func ExampleClient_metrics() { forUnitTestsRunInMockedContext(mockClient_metrics, func() { + clientMetrics := grpcprom.NewClientMetrics() + prometheus.Register(clientMetrics) cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialOptions: []grpc.DialOption{ - grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor), - grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor), + grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), + grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), }, }) if err != nil { diff --git a/tests/integration/clientv3/metrics_test.go b/tests/integration/clientv3/metrics_test.go index 252ad6ec8..92959e3d1 100644 --- a/tests/integration/clientv3/metrics_test.go +++ b/tests/integration/clientv3/metrics_test.go @@ -16,6 +16,7 @@ package clientv3test import ( "bufio" + "bytes" "context" "errors" "io" @@ -27,9 +28,10 @@ import ( "testing" "time" - grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" + grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/expfmt" "google.golang.org/grpc" "go.etcd.io/etcd/client/pkg/v3/transport" @@ -75,11 +77,14 @@ func TestV3ClientMetrics(t *testing.T) { clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1}) defer clus.Terminate(t) + clientMetrics := grpcprom.NewClientMetrics() + prometheus.Register(clientMetrics) + cfg := clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialOptions: []grpc.DialOption{ - grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor), - grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor), + grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), + grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), }, } cli, cerr := integration2.NewClient(t, cfg) @@ -147,6 +152,24 @@ func sumCountersForMetricAndLabels(t *testing.T, url string, metricName string, } func getHTTPBodyAsLines(t *testing.T, url string) []string { + data := getHTTPBodyAsBytes(t, url) + + reader := bufio.NewReader(bytes.NewReader(data)) + var lines []string + for { + line, err := reader.ReadString('\n') + if err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("error reading: %v", err) + } + lines = append(lines, line) + } + return lines +} + +func getHTTPBodyAsBytes(t *testing.T, url string) []byte { cfgtls := transport.TLSInfo{} tr, err := transport.NewTransport(cfgtls, time.Second) if err != nil { @@ -162,21 +185,12 @@ func getHTTPBodyAsLines(t *testing.T, url string) []string { if err != nil { t.Fatalf("Error fetching: %v", err) } - - reader := bufio.NewReader(resp.Body) - var lines []string - for { - line, err := reader.ReadString('\n') - if err != nil { - if errors.Is(err, io.EOF) { - break - } - t.Fatalf("error reading: %v", err) - } - lines = append(lines, line) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Error reading http body: %v", err) } - resp.Body.Close() - return lines + return body } func TestAllMetricsGenerated(t *testing.T) { @@ -213,7 +227,7 @@ func TestAllMetricsGenerated(t *testing.T) { url := "unix://" + addr + "/metrics" - clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1}) + clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1, Metrics: "extensive"}) defer clus.Terminate(t) clientMetrics := grpcprom.NewClientMetrics() @@ -249,42 +263,26 @@ func TestAllMetricsGenerated(t *testing.T) { // Define the expected list of metrics expectedMetrics := []string{ "etcd_cluster_version", - "etcd_disk_backend_commit_duration_seconds_bucket", - "etcd_disk_backend_commit_duration_seconds_count", - "etcd_disk_backend_commit_duration_seconds_sum", - "etcd_disk_backend_defrag_duration_seconds_bucket", - "etcd_disk_backend_defrag_duration_seconds_count", - "etcd_disk_backend_defrag_duration_seconds_sum", - "etcd_disk_backend_snapshot_duration_seconds_bucket", - "etcd_disk_backend_snapshot_duration_seconds_count", - "etcd_disk_backend_snapshot_duration_seconds_sum", + "etcd_disk_backend_commit_duration_seconds", + "etcd_disk_backend_defrag_duration_seconds", + "etcd_disk_backend_snapshot_duration_seconds", "etcd_disk_defrag_inflight", - "etcd_disk_wal_fsync_duration_seconds_bucket", - "etcd_disk_wal_fsync_duration_seconds_count", - "etcd_disk_wal_fsync_duration_seconds_sum", + "etcd_disk_wal_fsync_duration_seconds", "etcd_disk_wal_write_bytes_total", - "etcd_disk_wal_write_duration_seconds_bucket", - "etcd_disk_wal_write_duration_seconds_count", - "etcd_disk_wal_write_duration_seconds_sum", + "etcd_disk_wal_write_duration_seconds", "etcd_mvcc_db_open_read_transactions", "etcd_mvcc_db_total_size_in_bytes", "etcd_mvcc_db_total_size_in_use_in_bytes", "etcd_mvcc_delete_total", - "etcd_mvcc_hash_duration_seconds_bucket", - "etcd_mvcc_hash_duration_seconds_count", - "etcd_mvcc_hash_duration_seconds_sum", - "etcd_mvcc_hash_rev_duration_seconds_bucket", - "etcd_mvcc_hash_rev_duration_seconds_count", - "etcd_mvcc_hash_rev_duration_seconds_sum", + "etcd_mvcc_hash_duration_seconds", + "etcd_mvcc_hash_rev_duration_seconds", "etcd_mvcc_put_total", "etcd_mvcc_range_total", "etcd_mvcc_txn_total", "etcd_network_client_grpc_received_bytes_total", "etcd_network_client_grpc_sent_bytes_total", "etcd_network_known_peers", - "etcd_server_apply_duration_seconds_bucket", - "etcd_server_apply_duration_seconds_count", - "etcd_server_apply_duration_seconds_sum", + "etcd_server_apply_duration_seconds", "etcd_server_client_requests_total", "etcd_server_go_version", "etcd_server_has_leader", @@ -306,20 +304,15 @@ func TestAllMetricsGenerated(t *testing.T) { "etcd_server_slow_read_indexes_total", "etcd_server_snapshot_apply_in_progress_total", "etcd_server_version", - "etcd_snap_db_fsync_duration_seconds_bucket", - "etcd_snap_db_fsync_duration_seconds_count", - "etcd_snap_db_fsync_duration_seconds_sum", - "etcd_snap_db_save_total_duration_seconds_bucket", - "etcd_snap_db_save_total_duration_seconds_count", - "etcd_snap_db_save_total_duration_seconds_sum", - "etcd_snap_fsync_duration_seconds_bucket", - "etcd_snap_fsync_duration_seconds_count", - "etcd_snap_fsync_duration_seconds_sum", + "etcd_snap_db_fsync_duration_seconds", + "etcd_snap_db_save_total_duration_seconds", + "etcd_snap_fsync_duration_seconds", "grpc_client_handled_total", "grpc_client_msg_received_total", "grpc_client_msg_sent_total", "grpc_client_started_total", "grpc_server_handled_total", + "grpc_server_handling_seconds", "grpc_server_msg_received_total", "grpc_server_msg_sent_total", "grpc_server_started_total", @@ -335,20 +328,15 @@ func TestAllMetricsGenerated(t *testing.T) { } func getMetricsList(t *testing.T, url string) []string { - lines := getHTTPBodyAsLines(t, url) - metrics := make(map[string]struct{}) - for _, line := range lines { - if strings.Contains(line, "{") { - metric := line[:strings.Index(line, "{")] - metrics[metric] = struct{}{} - } else { - metric := line[:strings.Index(line, " ")] - metrics[metric] = struct{}{} - } + data := getHTTPBodyAsBytes(t, url) + var parser expfmt.TextParser + mfs, err := parser.TextToMetricFamilies(bytes.NewReader(data)) + if err != nil { + t.Errorf("Failed to parse metric families") } - var metricList []string - for metric := range metrics { - metricList = append(metricList, metric) + var ms []string + for key := range mfs { + ms = append(ms, key) } - return metricList + return ms }