From a9b8cba6021b0d2cc3e5df7bb5b95450af6ac5b8 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Tue, 11 Feb 2025 15:08:54 +0000 Subject: [PATCH] Migrate experimental distributed tracing flags to non-experimental flags Signed-off-by: Benjamin Wang --- server/config/config.go | 8 +- server/embed/config.go | 68 ++++++-- server/embed/config_tracing.go | 16 +- server/embed/etcd.go | 122 +++++++------- server/etcdmain/config.go | 25 +++ server/etcdmain/config_test.go | 240 ++++++++++++++++++++++++++++ server/etcdmain/help.go | 14 +- server/etcdserver/api/v3rpc/grpc.go | 6 +- tests/integration/tracing_test.go | 9 +- 9 files changed, 417 insertions(+), 91 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 8621feb80039..209f386cd545 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -167,10 +167,10 @@ type ServerConfig struct { EnableGRPCGateway bool - // ExperimentalEnableDistributedTracing enables distributed tracing using OpenTelemetry protocol. - ExperimentalEnableDistributedTracing bool - // ExperimentalTracerOptions are options for OpenTelemetry gRPC interceptor. - ExperimentalTracerOptions []otelgrpc.Option + // EnableDistributedTracing enables distributed tracing using OpenTelemetry protocol. + EnableDistributedTracing bool + // TracerOptions are options for OpenTelemetry gRPC interceptor. + TracerOptions []otelgrpc.Option WatchProgressNotifyInterval time.Duration diff --git a/server/embed/config.go b/server/embed/config.go index 05813b39a81a..d84575e41b19 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -102,9 +102,17 @@ const ( DefaultLogRotationConfig = `{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}` // ExperimentalDistributedTracingAddress is the default collector address. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use DefaultDistributedTracingAddress instead. + // TODO: delete in v3.7 ExperimentalDistributedTracingAddress = "localhost:4317" + // DefaultDistributedTracingAddress is the default collector address. + DefaultDistributedTracingAddress = "localhost:4317" // ExperimentalDistributedTracingServiceName is the default etcd service name. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use DefaultDistributedTracingServiceName instead. + // TODO: delete in v3.7 ExperimentalDistributedTracingServiceName = "etcd" + // DefaultDistributedTracingServiceName is the default etcd service name. + DefaultDistributedTracingServiceName = "etcd" DefaultExperimentalTxnModeWriteWithSharedBuffer = true @@ -151,6 +159,11 @@ var ( "experimental-compaction-sleep-interval": "compaction-sleep-interval", "experimental-downgrade-check-time": "downgrade-check-time", "experimental-peer-skip-client-san-verification": "peer-skip-client-san-verification", + "experimental-enable-distributed-tracing": "enable-distributed-tracing", + "experimental-distributed-tracing-address": "distributed-tracing-address", + "experimental-distributed-tracing-service-name": "distributed-tracing-service-name", + "experimental-distributed-tracing-instance-id": "distributed-tracing-instance-id", + "experimental-distributed-tracing-sampling-rate": "distributed-tracing-sampling-rate", } ) @@ -451,21 +464,47 @@ type Config struct { ListenMetricsUrlsJSON string `json:"listen-metrics-urls"` // ExperimentalEnableDistributedTracing indicates if experimental tracing using OpenTelemetry is enabled. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use EnableDistributedTracing instead. + // TODO: delete in v3.7 ExperimentalEnableDistributedTracing bool `json:"experimental-enable-distributed-tracing"` + // EnableDistributedTracing indicates if tracing using OpenTelemetry is enabled. + EnableDistributedTracing bool `json:"enable-distributed-tracing"` // ExperimentalDistributedTracingAddress is the address of the OpenTelemetry Collector. // Can only be set if ExperimentalEnableDistributedTracing is true. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use DistributedTracingAddress instead. + // TODO: delete in v3.7 ExperimentalDistributedTracingAddress string `json:"experimental-distributed-tracing-address"` + // DistributedTracingAddress is the address of the OpenTelemetry Collector. + // Can only be set if EnableDistributedTracing is true. + DistributedTracingAddress string `json:"distributed-tracing-address"` // ExperimentalDistributedTracingServiceName is the name of the service. // Can only be used if ExperimentalEnableDistributedTracing is true. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use DistributedTracingServiceName instead. + // TODO: delete in v3.7 ExperimentalDistributedTracingServiceName string `json:"experimental-distributed-tracing-service-name"` + // DistributedTracingServiceName is the name of the service. + // Can only be used if EnableDistributedTracing is true. + DistributedTracingServiceName string `json:"distributed-tracing-service-name"` // ExperimentalDistributedTracingServiceInstanceID is the ID key of the service. // This ID must be unique, as helps to distinguish instances of the same service // that exist at the same time. // Can only be used if ExperimentalEnableDistributedTracing is true. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use DistributedTracingServiceInstanceID instead. + // TODO: delete in v3.7 ExperimentalDistributedTracingServiceInstanceID string `json:"experimental-distributed-tracing-instance-id"` + // DistributedTracingServiceInstanceID is the ID key of the service. + // This ID must be unique, as helps to distinguish instances of the same service + // that exist at the same time. + // Can only be used if EnableDistributedTracing is true. + DistributedTracingServiceInstanceID string `json:"distributed-tracing-instance-id"` // ExperimentalDistributedTracingSamplingRatePerMillion is the number of samples to collect per million spans. // Defaults to 0. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use DistributedTracingSamplingRatePerMillion instead. + // TODO: delete in v3.7 ExperimentalDistributedTracingSamplingRatePerMillion int `json:"experimental-distributed-tracing-sampling-rate"` + // DistributedTracingSamplingRatePerMillion is the number of samples to collect per million spans. + // Defaults to 0. + DistributedTracingSamplingRatePerMillion int `json:"distributed-tracing-sampling-rate"` // ExperimentalPeerSkipClientSanVerification determines whether to skip verification of SAN field // in client certificate for peer connections. @@ -656,8 +695,10 @@ func NewConfig() *Config { ExperimentalMaxLearners: membership.DefaultMaxLearners, ExperimentalTxnModeWriteWithSharedBuffer: DefaultExperimentalTxnModeWriteWithSharedBuffer, - ExperimentalDistributedTracingAddress: ExperimentalDistributedTracingAddress, - ExperimentalDistributedTracingServiceName: ExperimentalDistributedTracingServiceName, + ExperimentalDistributedTracingAddress: DefaultDistributedTracingAddress, + DistributedTracingAddress: DefaultDistributedTracingAddress, + ExperimentalDistributedTracingServiceName: DefaultDistributedTracingServiceName, + DistributedTracingServiceName: DefaultDistributedTracingServiceName, CompactHashCheckTime: DefaultCompactHashCheckTime, // TODO: delete in v3.7 @@ -830,11 +871,20 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.Metrics, "metrics", cfg.Metrics, "Set level of detail for exported metrics, specify 'extensive' to include server side grpc histogram metrics") // experimental distributed tracing - fs.BoolVar(&cfg.ExperimentalEnableDistributedTracing, "experimental-enable-distributed-tracing", false, "Enable experimental distributed tracing using OpenTelemetry Tracing.") - fs.StringVar(&cfg.ExperimentalDistributedTracingAddress, "experimental-distributed-tracing-address", ExperimentalDistributedTracingAddress, "Address for distributed tracing used for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag).") - fs.StringVar(&cfg.ExperimentalDistributedTracingServiceName, "experimental-distributed-tracing-service-name", ExperimentalDistributedTracingServiceName, "Configures service name for distributed tracing to be used to define service name for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag). 'etcd' is the default service name. Use the same service name for all instances of etcd.") - fs.StringVar(&cfg.ExperimentalDistributedTracingServiceInstanceID, "experimental-distributed-tracing-instance-id", "", "Configures service instance ID for distributed tracing to be used to define service instance ID key for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag). There is no default value set. This ID must be unique per etcd instance.") - fs.IntVar(&cfg.ExperimentalDistributedTracingSamplingRatePerMillion, "experimental-distributed-tracing-sampling-rate", 0, "Number of samples to collect per million spans for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag).") + fs.BoolVar(&cfg.ExperimentalEnableDistributedTracing, "experimental-enable-distributed-tracing", false, "Enable experimental distributed tracing using OpenTelemetry Tracing. Deprecated in v3.6 and will be decommissioned in v3.7. Use --enable-distributed-tracing instead.") + fs.BoolVar(&cfg.EnableDistributedTracing, "enable-distributed-tracing", false, "Enable distributed tracing using OpenTelemetry Tracing.") + + fs.StringVar(&cfg.ExperimentalDistributedTracingAddress, "experimental-distributed-tracing-address", cfg.ExperimentalDistributedTracingAddress, "Address for distributed tracing used for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag). Deprecated in v3.6 and will be decommissioned in v3.7. Use --distributed-tracing-address instead.") + fs.StringVar(&cfg.DistributedTracingAddress, "distributed-tracing-address", cfg.DistributedTracingAddress, "Address for distributed tracing used for OpenTelemetry Tracing (if enabled with enable-distributed-tracing flag).") + + fs.StringVar(&cfg.ExperimentalDistributedTracingServiceName, "experimental-distributed-tracing-service-name", cfg.ExperimentalDistributedTracingServiceName, "Configures service name for distributed tracing to be used to define service name for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag). 'etcd' is the default service name. Use the same service name for all instances of etcd. Deprecated in v3.6 and will be decommissioned in v3.7. Use --distributed-tracing-service-name instead.") + fs.StringVar(&cfg.DistributedTracingServiceName, "distributed-tracing-service-name", cfg.DistributedTracingServiceName, "Configures service name for distributed tracing to be used to define service name for OpenTelemetry Tracing (if enabled with enable-distributed-tracing flag). 'etcd' is the default service name. Use the same service name for all instances of etcd.") + + fs.StringVar(&cfg.ExperimentalDistributedTracingServiceInstanceID, "experimental-distributed-tracing-instance-id", "", "Configures service instance ID for distributed tracing to be used to define service instance ID key for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag). There is no default value set. This ID must be unique per etcd instance. Deprecated in v3.6 and will be decommissioned in v3.7. Use --distributed-tracing-instance-id instead.") + fs.StringVar(&cfg.DistributedTracingServiceInstanceID, "distributed-tracing-instance-id", "", "Configures service instance ID for distributed tracing to be used to define service instance ID key for OpenTelemetry Tracing (if enabled with enable-distributed-tracing flag). There is no default value set. This ID must be unique per etcd instance.") + + fs.IntVar(&cfg.ExperimentalDistributedTracingSamplingRatePerMillion, "experimental-distributed-tracing-sampling-rate", 0, "Number of samples to collect per million spans for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag). Deprecated in v3.6 and will be decommissioned in v3.7. Use --distributed-tracing-sampling-rate instead.") + fs.IntVar(&cfg.DistributedTracingSamplingRatePerMillion, "distributed-tracing-sampling-rate", 0, "Number of samples to collect per million spans for OpenTelemetry Tracing (if enabled with enable-distributed-tracing flag).") // auth fs.StringVar(&cfg.AuthToken, "auth-token", cfg.AuthToken, "Specify auth token specific options.") @@ -1206,8 +1256,8 @@ func (cfg *Config) Validate() error { } // Validate distributed tracing configuration but only if enabled. - if cfg.ExperimentalEnableDistributedTracing { - if err := validateTracingConfig(cfg.ExperimentalDistributedTracingSamplingRatePerMillion); err != nil { + if cfg.EnableDistributedTracing { + if err := validateTracingConfig(cfg.DistributedTracingSamplingRatePerMillion); err != nil { return fmt.Errorf("distributed tracing configurition is not valid: (%w)", err) } } diff --git a/server/embed/config_tracing.go b/server/embed/config_tracing.go index 7fd86e8610f0..0ca90fdc52af 100644 --- a/server/embed/config_tracing.go +++ b/server/embed/config_tracing.go @@ -49,7 +49,7 @@ type tracingExporter struct { func newTracingExporter(ctx context.Context, cfg *Config) (*tracingExporter, error) { exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure(), - otlptracegrpc.WithEndpoint(cfg.ExperimentalDistributedTracingAddress), + otlptracegrpc.WithEndpoint(cfg.DistributedTracingAddress), ) if err != nil { return nil, err @@ -57,14 +57,14 @@ func newTracingExporter(ctx context.Context, cfg *Config) (*tracingExporter, err res, err := resource.New(ctx, resource.WithAttributes( - semconv.ServiceNameKey.String(cfg.ExperimentalDistributedTracingServiceName), + semconv.ServiceNameKey.String(cfg.DistributedTracingServiceName), ), ) if err != nil { return nil, err } - if resWithIDKey := determineResourceWithIDKey(cfg.ExperimentalDistributedTracingServiceInstanceID); resWithIDKey != nil { + if resWithIDKey := determineResourceWithIDKey(cfg.DistributedTracingServiceInstanceID); resWithIDKey != nil { // Merge resources into a new // resource in case of duplicates. res, err = resource.Merge(res, resWithIDKey) @@ -77,7 +77,7 @@ func newTracingExporter(ctx context.Context, cfg *Config) (*tracingExporter, err tracesdk.WithBatcher(exporter), tracesdk.WithResource(res), tracesdk.WithSampler( - tracesdk.ParentBased(determineSampler(cfg.ExperimentalDistributedTracingSamplingRatePerMillion)), + tracesdk.ParentBased(determineSampler(cfg.DistributedTracingSamplingRatePerMillion)), ), ) @@ -95,10 +95,10 @@ func newTracingExporter(ctx context.Context, cfg *Config) (*tracingExporter, err cfg.logger.Debug( "distributed tracing enabled", - zap.String("address", cfg.ExperimentalDistributedTracingAddress), - zap.String("service-name", cfg.ExperimentalDistributedTracingServiceName), - zap.String("service-instance-id", cfg.ExperimentalDistributedTracingServiceInstanceID), - zap.Int("sampling-rate", cfg.ExperimentalDistributedTracingSamplingRatePerMillion), + zap.String("address", cfg.DistributedTracingAddress), + zap.String("service-name", cfg.DistributedTracingServiceName), + zap.String("service-instance-id", cfg.DistributedTracingServiceInstanceID), + zap.Int("sampling-rate", cfg.DistributedTracingSamplingRatePerMillion), ) return &tracingExporter{ diff --git a/server/embed/etcd.go b/server/embed/etcd.go index 0e22388ad7b2..dae44d8ca95d 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -180,66 +180,66 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { backendFreelistType := parseBackendFreelistType(cfg.BackendFreelistType) srvcfg := config.ServerConfig{ - Name: cfg.Name, - ClientURLs: cfg.AdvertiseClientUrls, - PeerURLs: cfg.AdvertisePeerUrls, - DataDir: cfg.Dir, - DedicatedWALDir: cfg.WalDir, - SnapshotCount: cfg.SnapshotCount, - SnapshotCatchUpEntries: cfg.SnapshotCatchUpEntries, - MaxSnapFiles: cfg.MaxSnapFiles, - MaxWALFiles: cfg.MaxWalFiles, - InitialPeerURLsMap: urlsmap, - InitialClusterToken: token, - DiscoveryURL: cfg.Durl, - DiscoveryProxy: cfg.Dproxy, - DiscoveryCfg: cfg.DiscoveryCfg, - NewCluster: cfg.IsNewCluster(), - PeerTLSInfo: cfg.PeerTLSInfo, - TickMs: cfg.TickMs, - ElectionTicks: cfg.ElectionTicks(), - InitialElectionTickAdvance: cfg.InitialElectionTickAdvance, - AutoCompactionRetention: autoCompactionRetention, - AutoCompactionMode: cfg.AutoCompactionMode, - QuotaBackendBytes: cfg.QuotaBackendBytes, - BackendBatchLimit: cfg.BackendBatchLimit, - BackendFreelistType: backendFreelistType, - BackendBatchInterval: cfg.BackendBatchInterval, - MaxTxnOps: cfg.MaxTxnOps, - MaxRequestBytes: cfg.MaxRequestBytes, - MaxConcurrentStreams: cfg.MaxConcurrentStreams, - SocketOpts: cfg.SocketOpts, - StrictReconfigCheck: cfg.StrictReconfigCheck, - ClientCertAuthEnabled: cfg.ClientTLSInfo.ClientCertAuth, - AuthToken: cfg.AuthToken, - BcryptCost: cfg.BcryptCost, - TokenTTL: cfg.AuthTokenTTL, - CORS: cfg.CORS, - HostWhitelist: cfg.HostWhitelist, - CorruptCheckTime: cfg.CorruptCheckTime, - CompactHashCheckTime: cfg.CompactHashCheckTime, - PreVote: cfg.PreVote, - Logger: cfg.logger, - ForceNewCluster: cfg.ForceNewCluster, - EnableGRPCGateway: cfg.EnableGRPCGateway, - ExperimentalEnableDistributedTracing: cfg.ExperimentalEnableDistributedTracing, - UnsafeNoFsync: cfg.UnsafeNoFsync, - CompactionBatchLimit: cfg.CompactionBatchLimit, - CompactionSleepInterval: cfg.CompactionSleepInterval, - WatchProgressNotifyInterval: cfg.WatchProgressNotifyInterval, - DowngradeCheckTime: cfg.DowngradeCheckTime, - WarningApplyDuration: cfg.WarningApplyDuration, - WarningUnaryRequestDuration: cfg.WarningUnaryRequestDuration, - MemoryMlock: cfg.MemoryMlock, - BootstrapDefragThresholdMegabytes: cfg.BootstrapDefragThresholdMegabytes, - MaxLearners: cfg.MaxLearners, - V2Deprecation: cfg.V2DeprecationEffective(), - ExperimentalLocalAddress: cfg.InferLocalAddr(), - ServerFeatureGate: cfg.ServerFeatureGate, - Metrics: cfg.Metrics, - } - - if srvcfg.ExperimentalEnableDistributedTracing { + Name: cfg.Name, + ClientURLs: cfg.AdvertiseClientUrls, + PeerURLs: cfg.AdvertisePeerUrls, + DataDir: cfg.Dir, + DedicatedWALDir: cfg.WalDir, + SnapshotCount: cfg.SnapshotCount, + SnapshotCatchUpEntries: cfg.SnapshotCatchUpEntries, + MaxSnapFiles: cfg.MaxSnapFiles, + MaxWALFiles: cfg.MaxWalFiles, + InitialPeerURLsMap: urlsmap, + InitialClusterToken: token, + DiscoveryURL: cfg.Durl, + DiscoveryProxy: cfg.Dproxy, + DiscoveryCfg: cfg.DiscoveryCfg, + NewCluster: cfg.IsNewCluster(), + PeerTLSInfo: cfg.PeerTLSInfo, + TickMs: cfg.TickMs, + ElectionTicks: cfg.ElectionTicks(), + InitialElectionTickAdvance: cfg.InitialElectionTickAdvance, + AutoCompactionRetention: autoCompactionRetention, + AutoCompactionMode: cfg.AutoCompactionMode, + QuotaBackendBytes: cfg.QuotaBackendBytes, + BackendBatchLimit: cfg.BackendBatchLimit, + BackendFreelistType: backendFreelistType, + BackendBatchInterval: cfg.BackendBatchInterval, + MaxTxnOps: cfg.MaxTxnOps, + MaxRequestBytes: cfg.MaxRequestBytes, + MaxConcurrentStreams: cfg.MaxConcurrentStreams, + SocketOpts: cfg.SocketOpts, + StrictReconfigCheck: cfg.StrictReconfigCheck, + ClientCertAuthEnabled: cfg.ClientTLSInfo.ClientCertAuth, + AuthToken: cfg.AuthToken, + BcryptCost: cfg.BcryptCost, + TokenTTL: cfg.AuthTokenTTL, + CORS: cfg.CORS, + HostWhitelist: cfg.HostWhitelist, + CorruptCheckTime: cfg.CorruptCheckTime, + CompactHashCheckTime: cfg.CompactHashCheckTime, + PreVote: cfg.PreVote, + Logger: cfg.logger, + ForceNewCluster: cfg.ForceNewCluster, + EnableGRPCGateway: cfg.EnableGRPCGateway, + EnableDistributedTracing: cfg.EnableDistributedTracing, + UnsafeNoFsync: cfg.UnsafeNoFsync, + CompactionBatchLimit: cfg.CompactionBatchLimit, + CompactionSleepInterval: cfg.CompactionSleepInterval, + WatchProgressNotifyInterval: cfg.WatchProgressNotifyInterval, + DowngradeCheckTime: cfg.DowngradeCheckTime, + WarningApplyDuration: cfg.WarningApplyDuration, + WarningUnaryRequestDuration: cfg.WarningUnaryRequestDuration, + MemoryMlock: cfg.MemoryMlock, + BootstrapDefragThresholdMegabytes: cfg.BootstrapDefragThresholdMegabytes, + MaxLearners: cfg.MaxLearners, + V2Deprecation: cfg.V2DeprecationEffective(), + ExperimentalLocalAddress: cfg.InferLocalAddr(), + ServerFeatureGate: cfg.ServerFeatureGate, + Metrics: cfg.Metrics, + } + + if srvcfg.EnableDistributedTracing { tctx := context.Background() tracingExporter, terr := newTracingExporter(tctx, cfg) if terr != nil { @@ -248,7 +248,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { e.tracingExporterShutdown = func() { tracingExporter.Close(tctx) } - srvcfg.ExperimentalTracerOptions = tracingExporter.opts + srvcfg.TracerOptions = tracingExporter.opts e.cfg.logger.Info( "distributed tracing setup enabled", diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index 9905009df8d1..3738b8388c74 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -74,6 +74,11 @@ var ( "experimental-memory-mlock": "--experimental-memory-mlock is deprecated in v3.6 and will be decommissioned in v3.7. Use '--memory-mlock' instead.", "experimental-compaction-sleep-interval": "--experimental-compaction-sleep-interval is deprecated in v3.6 and will be decommissioned in v3.7. Use 'compaction-sleep-interval' instead.", "experimental-downgrade-check-time": "--experimental-downgrade-check-time is deprecated in v3.6 and will be decommissioned in v3.7. Use '--downgrade-check-time' instead.", + "experimental-enable-distributed-tracing": "--experimental-enable-distributed-tracing is deprecated in 3.6 and will be decommissioned in 3.7. Use --enable-distributed-tracing instead.", + "experimental-distributed-tracing-address": "--experimental-distributed-tracing-address is deprecated in 3.6 and will be decommissioned in 3.7. Use --distributed-tracing-address instead.", + "experimental-distributed-tracing-service-name": "--experimental-distributed-tracing-service-name is deprecated in 3.6 and will be decommissioned in 3.7. Use --distributed-tracing-service-name instead.", + "experimental-distributed-tracing-instance-id": "--experimental-distributed-tracing-instance-id is deprecated in 3.6 and will be decommissioned in 3.7. Use --distributed-tracing-instance-id instead.", + "experimental-distributed-tracing-sampling-rate": "--experimental-distributed-tracing-sampling-rate is deprecated in 3.6 and will be decommissioned in 3.7. Use --distributed-tracing-sampling-rate instead.", } ) @@ -222,6 +227,26 @@ func (cfg *config) parse(arguments []string) error { cfg.ec.DowngradeCheckTime = cfg.ec.ExperimentalDowngradeCheckTime } + if cfg.ec.FlagsExplicitlySet["experimental-enable-distributed-tracing"] { + cfg.ec.EnableDistributedTracing = cfg.ec.ExperimentalEnableDistributedTracing + } + + if cfg.ec.FlagsExplicitlySet["experimental-distributed-tracing-address"] { + cfg.ec.DistributedTracingAddress = cfg.ec.ExperimentalDistributedTracingAddress + } + + if cfg.ec.FlagsExplicitlySet["experimental-distributed-tracing-service-name"] { + cfg.ec.DistributedTracingServiceName = cfg.ec.ExperimentalDistributedTracingServiceName + } + + if cfg.ec.FlagsExplicitlySet["experimental-distributed-tracing-instance-id"] { + cfg.ec.DistributedTracingServiceInstanceID = cfg.ec.ExperimentalDistributedTracingServiceInstanceID + } + + if cfg.ec.FlagsExplicitlySet["experimental-distributed-tracing-sampling-rate"] { + cfg.ec.DistributedTracingSamplingRatePerMillion = cfg.ec.ExperimentalDistributedTracingSamplingRatePerMillion + } + // `V2Deprecation` (--v2-deprecation) is deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input. cfg.ec.V2Deprecation = cconfig.V2DeprDefault diff --git a/server/etcdmain/config_test.go b/server/etcdmain/config_test.go index aa4aea00768c..5fe66a98d798 100644 --- a/server/etcdmain/config_test.go +++ b/server/etcdmain/config_test.go @@ -1464,3 +1464,243 @@ func TestPeerSkipClientSanVerificationFlagMigration(t *testing.T) { }) } } + +// TestDistributedTracingFlagsMigration tests the migration from +// --experimental-distributed-tracing-* to --distributed-tracing-* +// TODO: delete in v3.7 +func TestDistributedTracingFlagsMigration(t *testing.T) { + testCases := []struct { + name string + + enableDistributedTracing string + distributedTracingAddress string + distributedTracingServiceName string + distributedTracingServiceInstanceID string + distributedTracingSamplingRatePerMillion string + + experimentalEnableDistributedTracing string + experimentalDistributedTracingAddress string + experimentalDistributedTracingServiceName string + experimentalDistributedTracingServiceInstanceID string + experimentalDistributedTracingSamplingRatePerMillion string + + expectedEnableDistributedTracing bool + expectedDistributedTracingAddress string + expectedDistributedTracingServiceName string + expectedDistributedTracingServiceInstanceID string + expectedDistributedTracingSamplingRatePerMillion int + + expectErr bool + }{ + // cannot set both experimental flags and non-experimental flags + { + name: "cannot set both experimental flag and non experimental flag 1", + enableDistributedTracing: "true", + experimentalEnableDistributedTracing: "true", + expectErr: true, + }, + { + name: "cannot set both experimental flag and non experimental flag 2", + distributedTracingAddress: "localhost:4317", + experimentalDistributedTracingAddress: "localhost:4318", + expectErr: true, + }, + { + name: "cannot set both experimental flag and non experimental flag 3", + distributedTracingServiceName: "etcd1", + experimentalDistributedTracingServiceName: "etcd2", + expectErr: true, + }, + { + name: "cannot set both experimental flag and non experimental flag 4", + distributedTracingServiceInstanceID: "fakeID", + experimentalDistributedTracingServiceInstanceID: "fakeID", + expectErr: true, + }, + { + name: "cannot set both experimental flag and non experimental flag 5", + distributedTracingSamplingRatePerMillion: "100", + experimentalDistributedTracingSamplingRatePerMillion: "100", + expectErr: true, + }, + // can set either --experimental-enable-distributed-tracing or --enable-distributed-tracing + { + name: "can set experimental-enable-distributed-tracing to true", + experimentalEnableDistributedTracing: "true", + expectedEnableDistributedTracing: true, + }, + { + name: "can set experimental-enable-distributed-tracing to false", + experimentalEnableDistributedTracing: "false", + expectedEnableDistributedTracing: false, + }, + { + name: "can set enable-distributed-tracing to true", + enableDistributedTracing: "true", + expectedEnableDistributedTracing: true, + }, + { + name: "can set enable-distributed-tracing to false", + enableDistributedTracing: "false", + expectedEnableDistributedTracing: false, + }, + // can set either --experimental-distributed-tracing-address or --distributed-tracing-address + { + name: "can set experimental-distributed-tracing-address", + experimentalDistributedTracingAddress: "localhost:1234", + expectedDistributedTracingAddress: "localhost:1234", + }, + { + name: "can set distributed-tracing-address", + distributedTracingAddress: "localhost:1234", + expectedDistributedTracingAddress: "localhost:1234", + }, + // can set either --experimental-distributed-tracing-service-name or --distributed-tracing-service-name + { + name: "can set experimental-distributed-tracing-service-name", + experimentalDistributedTracingServiceName: "fakeSererName", + expectedDistributedTracingServiceName: "fakeSererName", + }, + { + name: "can set distributed-tracing-service-name", + distributedTracingServiceName: "fakeSererName", + expectedDistributedTracingServiceName: "fakeSererName", + }, + // can set either --experimental-distributed-tracing-instance-id or --distributed-tracing-instance-id + { + name: "can set experimental-distributed-tracing-instance-id", + experimentalDistributedTracingServiceInstanceID: "fakeID", + expectedDistributedTracingServiceInstanceID: "fakeID", + }, + { + name: "can set distributed-tracing-instance-id", + distributedTracingServiceInstanceID: "fakeID", + expectedDistributedTracingServiceInstanceID: "fakeID", + }, + // can set either --experimental-distributed-tracing-sampling-rate or --distributed-tracing-sampling-rate + { + name: "can set experimental-distributed-tracing-sampling-rate", + experimentalDistributedTracingSamplingRatePerMillion: "200", + expectedDistributedTracingSamplingRatePerMillion: 200, + }, + { + name: "can set distributed-tracing-sampling-rate", + distributedTracingSamplingRatePerMillion: "300", + expectedDistributedTracingSamplingRatePerMillion: 300, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cmdLineArgs := []string{} + yc := struct { + ExperimentalEnableDistributedTracing bool `json:"experimental-enable-distributed-tracing,omitempty"` + EnableDistributedTracing bool `json:"enable-distributed-tracing,omitempty"` + + ExperimentalDistributedTracingAddress string `json:"experimental-distributed-tracing-address,omitempty"` + DistributedTracingAddress string `json:"distributed-tracing-address,omitempty"` + + ExperimentalDistributedTracingServiceName string `json:"experimental-distributed-tracing-service-name,omitempty"` + DistributedTracingServiceName string `json:"distributed-tracing-service-name,omitempty"` + + ExperimentalDistributedTracingServiceInstanceID string `json:"experimental-distributed-tracing-instance-id,omitempty"` + DistributedTracingServiceInstanceID string `json:"distributed-tracing-instance-id,omitempty"` + + ExperimentalDistributedTracingSamplingRatePerMillion int `json:"experimental-distributed-tracing-sampling-rate,omitempty"` + DistributedTracingSamplingRatePerMillion int `json:"distributed-tracing-sampling-rate,omitempty"` + }{} + + // --enable-distributed-tracing and --experimental-enable-distributed-tracing + if tc.enableDistributedTracing != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--enable-distributed-tracing=%s", tc.enableDistributedTracing)) + val, err := strconv.ParseBool(tc.enableDistributedTracing) + require.NoError(t, err) + yc.EnableDistributedTracing = val + } + if tc.experimentalEnableDistributedTracing != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--experimental-enable-distributed-tracing=%s", tc.experimentalEnableDistributedTracing)) + val, err := strconv.ParseBool(tc.experimentalEnableDistributedTracing) + require.NoError(t, err) + yc.ExperimentalEnableDistributedTracing = val + } + + // --distributed-tracing-address and --experimental-distributed-tracing-address + if tc.distributedTracingAddress != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--distributed-tracing-address=%s", tc.distributedTracingAddress)) + yc.DistributedTracingAddress = tc.distributedTracingAddress + } + if tc.experimentalDistributedTracingAddress != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--experimental-distributed-tracing-address=%s", tc.experimentalDistributedTracingAddress)) + yc.ExperimentalDistributedTracingAddress = tc.experimentalDistributedTracingAddress + } + + // --distributed-tracing-service-name and --experimental-distributed-tracing-service-name + if tc.distributedTracingServiceName != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--distributed-tracing-service-name=%s", tc.distributedTracingServiceName)) + yc.DistributedTracingServiceName = tc.distributedTracingServiceName + } + if tc.experimentalDistributedTracingServiceName != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--experimental-distributed-tracing-service-name=%s", tc.experimentalDistributedTracingServiceName)) + yc.ExperimentalDistributedTracingServiceName = tc.experimentalDistributedTracingServiceName + } + + // --distributed-tracing-instance-id and --experimental-distributed-tracing-instance-id + if tc.distributedTracingServiceInstanceID != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--distributed-tracing-instance-id=%s", tc.distributedTracingServiceInstanceID)) + yc.DistributedTracingServiceInstanceID = tc.distributedTracingServiceInstanceID + } + if tc.experimentalDistributedTracingServiceInstanceID != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--experimental-distributed-tracing-instance-id=%s", tc.experimentalDistributedTracingServiceInstanceID)) + yc.ExperimentalDistributedTracingServiceInstanceID = tc.experimentalDistributedTracingServiceInstanceID + } + + // --distributed-tracing-sampling-rate and --experimental-distributed-tracing-sampling-rate + if tc.distributedTracingSamplingRatePerMillion != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--distributed-tracing-sampling-rate=%s", tc.distributedTracingSamplingRatePerMillion)) + val, err := strconv.ParseInt(tc.distributedTracingSamplingRatePerMillion, 10, 64) + require.NoError(t, err) + yc.DistributedTracingSamplingRatePerMillion = int(val) + } + if tc.experimentalDistributedTracingSamplingRatePerMillion != "" { + cmdLineArgs = append(cmdLineArgs, fmt.Sprintf("--experimental-distributed-tracing-sampling-rate=%s", tc.experimentalDistributedTracingSamplingRatePerMillion)) + val, err := strconv.ParseInt(tc.experimentalDistributedTracingSamplingRatePerMillion, 10, 64) + require.NoError(t, err) + yc.ExperimentalDistributedTracingSamplingRatePerMillion = int(val) + } + + cfgFromCmdLine, errFromCmdLine, cfgFromFile, errFromFile := generateCfgsFromFileAndCmdLine(t, yc, cmdLineArgs) + + if tc.expectErr { + if errFromCmdLine == nil || errFromFile == nil { + t.Fatalf("expect parse error, got errFromCmdLine=%v, errFromFile=%v", errFromCmdLine, errFromFile) + } + return + } + if errFromCmdLine != nil || errFromFile != nil { + t.Fatal("error parsing config") + } + + // verify the expected values + require.Equal(t, tc.expectedEnableDistributedTracing, cfgFromCmdLine.ec.EnableDistributedTracing) + require.Equal(t, tc.expectedEnableDistributedTracing, cfgFromFile.ec.EnableDistributedTracing) + + if tc.expectedDistributedTracingAddress != "" { + require.Equal(t, tc.expectedDistributedTracingAddress, cfgFromCmdLine.ec.DistributedTracingAddress) + require.Equal(t, tc.expectedDistributedTracingAddress, cfgFromFile.ec.DistributedTracingAddress) + } + + if tc.expectedDistributedTracingServiceName != "" { + require.Equal(t, tc.expectedDistributedTracingServiceName, cfgFromCmdLine.ec.DistributedTracingServiceName) + require.Equal(t, tc.expectedDistributedTracingServiceName, cfgFromFile.ec.DistributedTracingServiceName) + } + + if tc.expectedDistributedTracingServiceInstanceID != "" { + require.Equal(t, tc.expectedDistributedTracingServiceInstanceID, cfgFromCmdLine.ec.DistributedTracingServiceInstanceID) + require.Equal(t, tc.expectedDistributedTracingServiceInstanceID, cfgFromFile.ec.DistributedTracingServiceInstanceID) + } + + require.Equal(t, tc.expectedDistributedTracingSamplingRatePerMillion, cfgFromCmdLine.ec.DistributedTracingSamplingRatePerMillion) + require.Equal(t, tc.expectedDistributedTracingSamplingRatePerMillion, cfgFromFile.ec.DistributedTracingSamplingRatePerMillion) + }) + } +} diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index b1c1af9e701e..6be9f68b707f 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -264,15 +264,25 @@ Logging: Experimental distributed tracing: --experimental-enable-distributed-tracing 'false' - Enable experimental distributed tracing. + Enable experimental distributed tracing. Deprecated in v3.6 and will be decommissioned in v3.7. Use --enable-distributed-tracing instead. + --enable-distributed-tracing 'false' + Enable distributed tracing. --experimental-distributed-tracing-address 'localhost:4317' + Distributed tracing collector address. Deprecated in v3.6 and will be decommissioned in v3.7. Use --distributed-tracing-address instead. + --distributed-tracing-address 'localhost:4317' Distributed tracing collector address. --experimental-distributed-tracing-service-name 'etcd' + Distributed tracing service name, must be same across all etcd instances. Deprecated in v3.6 and will be decommissioned in v3.7. Use --distributed-tracing-service-name instead. + --distributed-tracing-service-name 'etcd' Distributed tracing service name, must be same across all etcd instances. --experimental-distributed-tracing-instance-id '' + Distributed tracing instance ID, must be unique per each etcd instance. Deprecated in v3.6 and will be decommissioned in v3.7. Use --distributed-tracing-instance-id instead. + --distributed-tracing-instance-id '' Distributed tracing instance ID, must be unique per each etcd instance. --experimental-distributed-tracing-sampling-rate '0' - Number of samples to collect per million spans for distributed tracing. Disabled by default. + Number of samples to collect per million spans for distributed tracing. Disabled by default. Deprecated in v3.6 and will be decommissioned in v3.7. Use --distributed-tracing-sampling-rate instead. + --distributed-tracing-sampling-rate '0' + Number of samples to collect per million spans for distributed tracing. Experimental feature: --experimental-initial-corrupt-check 'false'. It's deprecated, and will be decommissioned in v3.7. Use '--feature-gates=InitialCorruptCheck=true' instead. diff --git a/server/etcdserver/api/v3rpc/grpc.go b/server/etcdserver/api/v3rpc/grpc.go index 810fd219b8fa..ed55e0357c9f 100644 --- a/server/etcdserver/api/v3rpc/grpc.go +++ b/server/etcdserver/api/v3rpc/grpc.go @@ -66,9 +66,9 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer serverMetrics.StreamServerInterceptor(), } - if s.Cfg.ExperimentalEnableDistributedTracing { - chainUnaryInterceptors = append(chainUnaryInterceptors, otelgrpc.UnaryServerInterceptor(s.Cfg.ExperimentalTracerOptions...)) - chainStreamInterceptors = append(chainStreamInterceptors, otelgrpc.StreamServerInterceptor(s.Cfg.ExperimentalTracerOptions...)) + if s.Cfg.EnableDistributedTracing { + chainUnaryInterceptors = append(chainUnaryInterceptors, otelgrpc.UnaryServerInterceptor(s.Cfg.TracerOptions...)) + chainStreamInterceptors = append(chainStreamInterceptors, otelgrpc.StreamServerInterceptor(s.Cfg.TracerOptions...)) } opts = append(opts, grpc.ChainUnaryInterceptor(chainUnaryInterceptors...)) diff --git a/tests/integration/tracing_test.go b/tests/integration/tracing_test.go index d255e958308d..74f231a3aba4 100644 --- a/tests/integration/tracing_test.go +++ b/tests/integration/tracing_test.go @@ -55,10 +55,11 @@ func TestTracing(t *testing.T) { defer srv.Stop() cfg := integration.NewEmbedConfig(t, "default") - cfg.ExperimentalEnableDistributedTracing = true - cfg.ExperimentalDistributedTracingAddress = listener.Addr().String() - cfg.ExperimentalDistributedTracingServiceName = "integration-test-tracing" - cfg.ExperimentalDistributedTracingSamplingRatePerMillion = 100 + + cfg.EnableDistributedTracing = true + cfg.DistributedTracingAddress = listener.Addr().String() + cfg.DistributedTracingServiceName = "integration-test-tracing" + cfg.DistributedTracingSamplingRatePerMillion = 100 // start an etcd instance with tracing enabled etcdSrv, err := embed.StartEtcd(cfg)