From 5fb9a6e366abd090561328951fb9073568d08b64 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 09:46:47 +0300 Subject: [PATCH 01/36] rename variable mysqlMetricThresholds to metricThresholds Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/throttle/throttler.go | 32 +++++++++---------- .../tabletserver/throttle/throttler_test.go | 20 ++++++------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index e3e36123e7e..2535150e97a 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -192,12 +192,12 @@ type Throttler struct { MetricsThreshold atomic.Uint64 checkAsCheckSelf atomic.Bool - mysqlMetricThresholds *cache.Cache - aggregatedMetrics *cache.Cache - throttledApps *cache.Cache - recentApps *cache.Cache - metricsHealth *cache.Cache - appCheckedMetrics *cache.Cache + metricThresholds *cache.Cache + aggregatedMetrics *cache.Cache + throttledApps *cache.Cache + recentApps *cache.Cache + metricsHealth *cache.Cache + appCheckedMetrics *cache.Cache initMutex sync.Mutex enableMutex sync.Mutex @@ -258,7 +258,7 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv throttler.mysqlInventory = mysql.NewInventory() throttler.throttledApps = cache.New(cache.NoExpiration, 0) - throttler.mysqlMetricThresholds = cache.New(cache.NoExpiration, 0) + throttler.metricThresholds = cache.New(cache.NoExpiration, 0) throttler.aggregatedMetrics = cache.New(aggregatedMetricsExpiration, 0) throttler.recentApps = cache.New(recentAppsExpiration, recentAppsExpiration) throttler.metricsHealth = cache.New(cache.NoExpiration, 0) @@ -426,14 +426,14 @@ func (throttler *Throttler) WatchSrvKeyspaceCallback(srvks *topodatapb.SrvKeyspa // - throttler config. This can be a list of zero or more entries. These metrics override the inventory. func (throttler *Throttler) convergeMetricThresholds() { for _, metricName := range base.KnownMetricNames { - if val, ok := throttler.mysqlMetricThresholds.Get(throttlerConfigPrefix + metricName.String()); ok { + if val, ok := throttler.metricThresholds.Get(throttlerConfigPrefix + metricName.String()); ok { // Value supplied by throttler config takes precedence - throttler.mysqlMetricThresholds.Set(metricName.String(), val, cache.DefaultExpiration) + throttler.metricThresholds.Set(metricName.String(), val, cache.DefaultExpiration) continue } // metric not indicated in the throttler config, therefore we should use the default threshold for that metric - if val, ok := throttler.mysqlMetricThresholds.Get(inventoryPrefix + metricName.String()); ok { - throttler.mysqlMetricThresholds.Set(metricName.String(), val, cache.DefaultExpiration) + if val, ok := throttler.metricThresholds.Get(inventoryPrefix + metricName.String()); ok { + throttler.metricThresholds.Set(metricName.String(), val, cache.DefaultExpiration) } } } @@ -502,13 +502,13 @@ func (throttler *Throttler) applyThrottlerConfig(ctx context.Context, throttlerC { // Metric thresholds for metricName, threshold := range throttlerConfig.MetricThresholds { - throttler.mysqlMetricThresholds.Set(throttlerConfigPrefix+metricName, threshold, cache.DefaultExpiration) + throttler.metricThresholds.Set(throttlerConfigPrefix+metricName, threshold, cache.DefaultExpiration) } for _, metricName := range base.KnownMetricNames { if _, ok := throttlerConfig.MetricThresholds[metricName.String()]; !ok { // metric not indicated in the throttler config, therefore should be removed from the map // so that we know to apply the inventory default threshold - throttler.mysqlMetricThresholds.Delete(throttlerConfigPrefix + metricName.String()) + throttler.metricThresholds.Delete(throttlerConfigPrefix + metricName.String()) } } throttler.convergeMetricThresholds() @@ -1181,7 +1181,7 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { threshold = metricsThreshold } - throttler.mysqlMetricThresholds.Set(inventoryPrefix+metricName.String(), math.Float64frombits(threshold), cache.DefaultExpiration) + throttler.metricThresholds.Set(inventoryPrefix+metricName.String(), math.Float64frombits(threshold), cache.DefaultExpiration) } throttler.convergeMetricThresholds() clusterSettingsCopy := *mysqlSettings @@ -1284,7 +1284,7 @@ func (throttler *Throttler) getAggregatedMetric(aggregatedName string) base.Metr } func (throttler *Throttler) getMySQLStoreMetric(ctx context.Context, scope base.Scope, metricName base.MetricName) (base.MetricResult, float64) { - thresholdVal, found := throttler.mysqlMetricThresholds.Get(metricName.String()) + thresholdVal, found := throttler.metricThresholds.Get(metricName.String()) if !found { return base.NoSuchMetric, 0 } @@ -1295,7 +1295,7 @@ func (throttler *Throttler) getMySQLStoreMetric(ctx context.Context, scope base. func (throttler *Throttler) mysqlMetricThresholdsSnapshot() map[string]float64 { snapshot := make(map[string]float64) - for key, value := range throttler.mysqlMetricThresholds.Items() { + for key, value := range throttler.metricThresholds.Items() { threshold, _ := value.Object.(float64) snapshot[key] = threshold } diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index d9b910a3152..aba313c91cf 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -253,7 +253,7 @@ func newTestThrottler() *Throttler { throttler.mysqlInventory = mysql.NewInventory() throttler.throttledApps = cache.New(cache.NoExpiration, 0) - throttler.mysqlMetricThresholds = cache.New(cache.NoExpiration, 0) + throttler.metricThresholds = cache.New(cache.NoExpiration, 0) throttler.aggregatedMetrics = cache.New(10*aggregatedMetricsExpiration, 0) throttler.recentApps = cache.New(recentAppsExpiration, 0) throttler.metricsHealth = cache.New(cache.NoExpiration, 0) @@ -380,17 +380,17 @@ func TestApplyThrottlerConfig(t *testing.T) { }) t.Run("metric thresholds", func(t *testing.T) { { - val, ok := throttler.mysqlMetricThresholds.Get("lag") + val, ok := throttler.metricThresholds.Get("lag") require.True(t, ok) assert.Equal(t, float64(0.75), val) } { - val, ok := throttler.mysqlMetricThresholds.Get("threads_running") + val, ok := throttler.metricThresholds.Get("threads_running") require.True(t, ok) assert.Equal(t, float64(3.0), val) } { - val, ok := throttler.mysqlMetricThresholds.Get("loadavg") + val, ok := throttler.metricThresholds.Get("loadavg") require.True(t, ok) assert.Equal(t, float64(1.0), val) } @@ -432,7 +432,7 @@ func TestApplyThrottlerConfigMetricThresholds(t *testing.T) { t.Run("check low threshold", func(t *testing.T) { sleepTillThresholdApplies() { - _, ok := throttler.mysqlMetricThresholds.Get("config/lag") + _, ok := throttler.metricThresholds.Get("config/lag") assert.False(t, ok) } assert.Equal(t, float64(0.0033), throttler.GetMetricsThreshold()) @@ -455,7 +455,7 @@ func TestApplyThrottlerConfigMetricThresholds(t *testing.T) { t.Run("check with high 'lag' threshold", func(t *testing.T) { sleepTillThresholdApplies() { - val, ok := throttler.mysqlMetricThresholds.Get("config/lag") + val, ok := throttler.metricThresholds.Get("config/lag") require.True(t, ok) assert.Equal(t, float64(4444), val) } @@ -471,17 +471,17 @@ func TestApplyThrottlerConfigMetricThresholds(t *testing.T) { assert.Equal(t, float64(0.0033), throttler.GetMetricsThreshold()) t.Run("metric thresholds", func(t *testing.T) { { - val, ok := throttler.mysqlMetricThresholds.Get("config/lag") + val, ok := throttler.metricThresholds.Get("config/lag") require.True(t, ok) assert.Equal(t, float64(4444), val) } { - val, ok := throttler.mysqlMetricThresholds.Get("inventory/lag") + val, ok := throttler.metricThresholds.Get("inventory/lag") require.True(t, ok) assert.Equal(t, float64(0.0033), val) } { - val, ok := throttler.mysqlMetricThresholds.Get("lag") + val, ok := throttler.metricThresholds.Get("lag") require.True(t, ok) assert.Equal(t, float64(4444), val) } @@ -903,7 +903,7 @@ func TestRefreshMySQLInventory(t *testing.T) { throttler := &Throttler{ mysqlClusterProbesChan: make(chan *mysql.ClusterProbes), - mysqlMetricThresholds: cache.New(cache.NoExpiration, 0), + metricThresholds: cache.New(cache.NoExpiration, 0), ts: &FakeTopoServer{}, mysqlInventory: mysql.NewInventory(), } From c6d70474b796ebf3fa7cc1e4e29888f72faf0c29 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 09:47:07 +0300 Subject: [PATCH 02/36] rename function mysqlMetricThresholdsSnapshot to metricThresholdsSnapshot Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/throttler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 2535150e97a..042a8aa6c9f 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1293,7 +1293,7 @@ func (throttler *Throttler) getMySQLStoreMetric(ctx context.Context, scope base. return throttler.getAggregatedMetric(aggregatedName), threshold } -func (throttler *Throttler) mysqlMetricThresholdsSnapshot() map[string]float64 { +func (throttler *Throttler) metricThresholdsSnapshot() map[string]float64 { snapshot := make(map[string]float64) for key, value := range throttler.metricThresholds.Items() { threshold, _ := value.Object.(float64) @@ -1650,7 +1650,7 @@ func (throttler *Throttler) Status() *ThrottlerStatus { MetricNameUsedAsDefault: throttler.metricNameUsedAsDefault().String(), AggregatedMetrics: throttler.aggregatedMetricsSnapshot(), - MetricsThresholds: throttler.mysqlMetricThresholdsSnapshot(), + MetricsThresholds: throttler.metricThresholdsSnapshot(), MetricsHealth: throttler.metricsHealthSnapshot(), ThrottledApps: throttler.ThrottledApps(), AppCheckedMetrics: throttler.appCheckedMetricsSnapshot(), From 9eb59b312e4e64b8c8f2fc4e28cf361085161176 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 09:48:49 +0300 Subject: [PATCH 03/36] rename *Interval variables away from 'mysql' Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/throttle/throttler.go | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 042a8aa6c9f..dcd6b196b8f 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -85,10 +85,10 @@ import ( const ( leaderCheckInterval = 5 * time.Second - mysqlCollectInterval = 250 * time.Millisecond // PRIMARY polls replicas - mysqlDormantCollectInterval = 5 * time.Second // PRIMARY polls replicas when dormant (no recent checks) - mysqlRefreshInterval = 10 * time.Second // Refreshing tablet inventory - mysqlAggregateInterval = 125 * time.Millisecond + activeCollectInterval = 250 * time.Millisecond // PRIMARY polls replicas + dormantCollectInterval = 5 * time.Second // PRIMARY polls replicas when dormant (no recent checks) + inventoryRefreshInterval = 10 * time.Second // Refreshing tablet inventory + metricsAggregateInterval = 125 * time.Millisecond throttledAppsSnapshotInterval = 5 * time.Second recentCheckRateLimiterInterval = 1 * time.Second // Ticker assisting in determining when the throttler was last checked @@ -264,15 +264,15 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv throttler.metricsHealth = cache.New(cache.NoExpiration, 0) throttler.appCheckedMetrics = cache.New(cache.NoExpiration, 0) - throttler.httpClient = base.SetupHTTPClient(2 * mysqlCollectInterval) + throttler.httpClient = base.SetupHTTPClient(2 * activeCollectInterval) throttler.initThrottleTabletTypes() throttler.check = NewThrottlerCheck(throttler) throttler.leaderCheckInterval = leaderCheckInterval - throttler.mysqlCollectInterval = mysqlCollectInterval - throttler.mysqlDormantCollectInterval = mysqlDormantCollectInterval - throttler.mysqlRefreshInterval = mysqlRefreshInterval - throttler.mysqlAggregateInterval = mysqlAggregateInterval + throttler.mysqlCollectInterval = activeCollectInterval + throttler.mysqlDormantCollectInterval = dormantCollectInterval + throttler.mysqlRefreshInterval = inventoryRefreshInterval + throttler.mysqlAggregateInterval = metricsAggregateInterval throttler.throttledAppsSnapshotInterval = throttledAppsSnapshotInterval throttler.dormantPeriod = dormantPeriod throttler.recentCheckDormantDiff = int64(throttler.dormantPeriod / recentCheckRateLimiterInterval) @@ -1012,7 +1012,7 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie } return func(ctx context.Context) mysql.MySQLThrottleMetrics { // Some reasonable timeout, to ensure we release connections even if they're hanging (otherwise grpc-go keeps polling those connections forever) - ctx, cancel := context.WithTimeout(ctx, 4*mysqlCollectInterval) + ctx, cancel := context.WithTimeout(ctx, 4*activeCollectInterval) defer cancel() // Hit a tablet's `check-self` via HTTP, and convert its CheckResult JSON output into a MySQLThrottleMetric @@ -1205,7 +1205,7 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { // not the leader (primary tablet)? Then no more work for us. } // The primary tablet is also in charge of collecting the shard's metrics - ctx, cancel := context.WithTimeout(ctx, mysqlRefreshInterval) + ctx, cancel := context.WithTimeout(ctx, inventoryRefreshInterval) defer cancel() tabletAliases, err := throttler.ts.FindAllTabletAliasesInShard(ctx, throttler.keyspace, throttler.shard) From 9f2716af8e8459659334014b3d2b43db73b78a28 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 09:49:32 +0300 Subject: [PATCH 04/36] rename *Interval variables away from 'mysql' Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/throttle/throttler.go | 24 +++++++++---------- .../tabletserver/throttle/throttler_test.go | 8 +++---- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index dcd6b196b8f..2da4ee9475a 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -157,10 +157,10 @@ type Throttler struct { isOpen atomic.Bool leaderCheckInterval time.Duration - mysqlCollectInterval time.Duration - mysqlDormantCollectInterval time.Duration - mysqlRefreshInterval time.Duration - mysqlAggregateInterval time.Duration + activeCollectInterval time.Duration + dormantCollectInterval time.Duration + inventoryRefreshInterval time.Duration + metricsAggregateInterval time.Duration throttledAppsSnapshotInterval time.Duration dormantPeriod time.Duration @@ -269,10 +269,10 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv throttler.check = NewThrottlerCheck(throttler) throttler.leaderCheckInterval = leaderCheckInterval - throttler.mysqlCollectInterval = activeCollectInterval - throttler.mysqlDormantCollectInterval = dormantCollectInterval - throttler.mysqlRefreshInterval = inventoryRefreshInterval - throttler.mysqlAggregateInterval = metricsAggregateInterval + throttler.activeCollectInterval = activeCollectInterval + throttler.dormantCollectInterval = dormantCollectInterval + throttler.inventoryRefreshInterval = inventoryRefreshInterval + throttler.metricsAggregateInterval = metricsAggregateInterval throttler.throttledAppsSnapshotInterval = throttledAppsSnapshotInterval throttler.dormantPeriod = dormantPeriod throttler.recentCheckDormantDiff = int64(throttler.dormantPeriod / recentCheckRateLimiterInterval) @@ -862,10 +862,10 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { return t } leaderCheckTicker := addTicker(throttler.leaderCheckInterval) - mysqlCollectTicker := addTicker(throttler.mysqlCollectInterval) - mysqlDormantCollectTicker := addTicker(throttler.mysqlDormantCollectInterval) - mysqlRefreshTicker := addTicker(throttler.mysqlRefreshInterval) - mysqlAggregateTicker := addTicker(throttler.mysqlAggregateInterval) + mysqlCollectTicker := addTicker(throttler.activeCollectInterval) + mysqlDormantCollectTicker := addTicker(throttler.dormantCollectInterval) + mysqlRefreshTicker := addTicker(throttler.inventoryRefreshInterval) + mysqlAggregateTicker := addTicker(throttler.metricsAggregateInterval) throttledAppsTicker := addTicker(throttler.throttledAppsSnapshotInterval) primaryStimulatorRateLimiter := timer.NewRateLimiter(throttler.dormantPeriod) throttler.recentCheckRateLimiter = timer.NewRateLimiter(recentCheckRateLimiterInterval) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index aba313c91cf..f2151e3b50f 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -263,10 +263,10 @@ func newTestThrottler() *Throttler { // High contention & racy intervals: throttler.leaderCheckInterval = 10 * time.Millisecond - throttler.mysqlCollectInterval = 10 * time.Millisecond - throttler.mysqlDormantCollectInterval = 10 * time.Millisecond - throttler.mysqlRefreshInterval = 10 * time.Millisecond - throttler.mysqlAggregateInterval = 10 * time.Millisecond + throttler.activeCollectInterval = 10 * time.Millisecond + throttler.dormantCollectInterval = 10 * time.Millisecond + throttler.inventoryRefreshInterval = 10 * time.Millisecond + throttler.metricsAggregateInterval = 10 * time.Millisecond throttler.throttledAppsSnapshotInterval = 10 * time.Millisecond throttler.dormantPeriod = 5 * time.Second throttler.recentCheckDormantDiff = int64(throttler.dormantPeriod / recentCheckRateLimiterInterval) From 685441e2370aec26e29289d59dfa94098ea3d870 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:08:34 +0300 Subject: [PATCH 05/36] rename aggregateMySQLProbes -> aggregateMetricResults. Cleanup tests Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/mysql.go | 2 +- .../tabletserver/throttle/mysql_test.go | 57 ++++++++----------- .../tabletserver/throttle/throttler.go | 2 +- 3 files changed, 25 insertions(+), 36 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/mysql.go b/go/vt/vttablet/tabletserver/throttle/mysql.go index ae0361c6018..ee24bf38b40 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql.go @@ -49,7 +49,7 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/mysql" ) -func aggregateMySQLProbes( +func aggregateMetricResults( ctx context.Context, metricName base.MetricName, tabletResultsMap mysql.TabletResultMap, diff --git a/go/vt/vttablet/tabletserver/throttle/mysql_test.go b/go/vt/vttablet/tabletserver/throttle/mysql_test.go index 78cc1fdbdd3..8fb25f0929d 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql_test.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql_test.go @@ -78,7 +78,7 @@ func noSuchMetricMap() base.MetricResultMap { return result } -func TestAggregateMySQLProbesNoErrors(t *testing.T) { +func TestAggregateMetricResultsNoErrors(t *testing.T) { ctx := context.Background() tabletResultsMap := mysql.TabletResultMap{ alias1: newMetricResultMap(1.2), @@ -87,50 +87,46 @@ func TestAggregateMySQLProbesNoErrors(t *testing.T) { alias4: newMetricResultMap(0.6), alias5: newMetricResultMap(1.1), } - var probes mysql.Probes = map[string](*mysql.Probe){} - for clusterKey := range tabletResultsMap { - probes[clusterKey] = &mysql.Probe{Alias: clusterKey} - } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.2) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.1) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 3, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 3, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 4, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 4, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.3) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 5, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 5, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.3) } } -func TestAggregateMySQLProbesNoErrorsIgnoreHostsThreshold(t *testing.T) { +func TestAggregateMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { ctx := context.Background() tabletResultsMap := mysql.TabletResultMap{ alias1: newMetricResultMap(1.2), @@ -139,49 +135,46 @@ func TestAggregateMySQLProbesNoErrorsIgnoreHostsThreshold(t *testing.T) { alias4: newMetricResultMap(0.6), alias5: newMetricResultMap(1.1), } - var probes mysql.Probes = map[string](*mysql.Probe){} - for clusterKey := range tabletResultsMap { - probes[clusterKey] = &mysql.Probe{Alias: clusterKey} - } + { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 1.0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 1.0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.2) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 1.0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.1) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 3, false, 1.0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 3, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 4, false, 1.0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 4, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 5, false, 1.0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 5, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } } -func TestAggregateMySQLProbesWithErrors(t *testing.T) { +func TestAggregateMetricResultsWithErrors(t *testing.T) { ctx := context.Background() tabletResultsMap := mysql.TabletResultMap{ alias1: newMetricResultMap(1.2), @@ -190,31 +183,27 @@ func TestAggregateMySQLProbesWithErrors(t *testing.T) { alias4: noSuchMetricMap(), alias5: newMetricResultMap(1.1), } - var probes mysql.Probes = map[string](*mysql.Probe){} - for clusterKey := range tabletResultsMap { - probes[clusterKey] = &mysql.Probe{Alias: clusterKey} - } t.Run("nonexistent", func(t *testing.T) { - worstMetric := aggregateMySQLProbes(ctx, nonexistentMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateMetricResults(ctx, nonexistentMetricName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("no ignore", func(t *testing.T) { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("ignore 1", func(t *testing.T) { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, 1.7, value) }) t.Run("ignore 2", func(t *testing.T) { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, 1.2, value) @@ -222,19 +211,19 @@ func TestAggregateMySQLProbesWithErrors(t *testing.T) { tabletResultsMap[alias1][base.DefaultMetricName] = base.NoSuchMetric t.Run("no such metric", func(t *testing.T) { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("no such metric, ignore 1", func(t *testing.T) { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("metric found", func(t *testing.T) { - worstMetric := aggregateMySQLProbes(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 0) + worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 2da4ee9475a..9a673bbfe54 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1255,7 +1255,7 @@ func (throttler *Throttler) aggregateMySQLMetrics(ctx context.Context) error { ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold ignoreDialTCPErrors := throttler.configSettings.MySQLStore.IgnoreDialTCPErrors - aggregatedMetric := aggregateMySQLProbes(ctx, metricName, tabletResultsMap, ignoreHostsCount, ignoreDialTCPErrors, ignoreHostsThreshold) + aggregatedMetric := aggregateMetricResults(ctx, metricName, tabletResultsMap, ignoreHostsCount, ignoreDialTCPErrors, ignoreHostsThreshold) aggregatedMetricName := metricName.AggregatedName(scope) throttler.aggregatedMetrics.Set(aggregatedMetricName, aggregatedMetric, cache.DefaultExpiration) if metricName == metricNameUsedAsDefault { From dd11de1b32542f5ee5eecd9f3dca6fe301bdc87c Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:16:00 +0300 Subject: [PATCH 06/36] rename MySQLThrottleMetrics -> ThrottleMetrics. Rename mysqlThrottleMetricChan -> throttleMetricChan Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../throttle/mysql/mysql_throttle_metric.go | 52 ++++++------- .../tabletserver/throttle/throttler.go | 74 +++++++++---------- .../tabletserver/throttle/throttler_test.go | 16 ++-- 3 files changed, 71 insertions(+), 71 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go b/go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go index 58447315a94..d4bd079572a 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go @@ -66,31 +66,31 @@ const ( MetricsQueryTypeUnknown ) -var mysqlMetricCache = cache.New(cache.NoExpiration, 10*time.Second) +var metricCache = cache.New(cache.NoExpiration, 10*time.Second) -func getMySQLMetricCacheKey(probe *Probe) string { +func getMetricCacheKey(probe *Probe) string { return probe.Alias } -func cacheMySQLThrottleMetric(probe *Probe, mySQLThrottleMetrics MySQLThrottleMetrics) MySQLThrottleMetrics { - for _, metric := range mySQLThrottleMetrics { +func cacheThrottleMetric(probe *Probe, throttleMetrics ThrottleMetrics) ThrottleMetrics { + for _, metric := range throttleMetrics { if metric.Err != nil { - return mySQLThrottleMetrics + return throttleMetrics } } if probe.CacheMillis > 0 { - mysqlMetricCache.Set(getMySQLMetricCacheKey(probe), mySQLThrottleMetrics, time.Duration(probe.CacheMillis)*time.Millisecond) + metricCache.Set(getMetricCacheKey(probe), throttleMetrics, time.Duration(probe.CacheMillis)*time.Millisecond) } - return mySQLThrottleMetrics + return throttleMetrics } -func getCachedMySQLThrottleMetrics(probe *Probe) MySQLThrottleMetrics { +func getCachedThrottleMetrics(probe *Probe) ThrottleMetrics { if probe.CacheMillis == 0 { return nil } - if metrics, found := mysqlMetricCache.Get(getMySQLMetricCacheKey(probe)); found { - mySQLThrottleMetrics, _ := metrics.(MySQLThrottleMetrics) - return mySQLThrottleMetrics + if metrics, found := metricCache.Get(getMetricCacheKey(probe)); found { + throttleMetrics, _ := metrics.(ThrottleMetrics) + return throttleMetrics } return nil } @@ -109,8 +109,8 @@ func GetMetricsQueryType(query string) MetricsQueryType { return MetricsQueryTypeUnknown } -// MySQLThrottleMetric has the probed metric for a tablet -type MySQLThrottleMetric struct { // nolint:revive +// ThrottleMetric has the probed metric for a tablet +type ThrottleMetric struct { // nolint:revive Name base.MetricName Scope base.Scope Alias string @@ -118,40 +118,40 @@ type MySQLThrottleMetric struct { // nolint:revive Err error } -type MySQLThrottleMetrics map[base.MetricName]*MySQLThrottleMetric // nolint:revive +type ThrottleMetrics map[base.MetricName]*ThrottleMetric // nolint:revive -// NewMySQLThrottleMetric creates a new MySQLThrottleMetric -func NewMySQLThrottleMetric() *MySQLThrottleMetric { - return &MySQLThrottleMetric{Value: 0} +// NewThrottleMetric creates a new ThrottleMetric +func NewThrottleMetric() *ThrottleMetric { + return &ThrottleMetric{Value: 0} } // GetClusterTablet returns the ClusterTablet part of the metric -func (metric *MySQLThrottleMetric) GetTabletAlias() string { +func (metric *ThrottleMetric) GetTabletAlias() string { return metric.Alias } // Get implements MetricResult -func (metric *MySQLThrottleMetric) Get() (float64, error) { +func (metric *ThrottleMetric) Get() (float64, error) { return metric.Value, metric.Err } // WithError returns this metric with given error -func (metric *MySQLThrottleMetric) WithError(err error) *MySQLThrottleMetric { +func (metric *ThrottleMetric) WithError(err error) *ThrottleMetric { metric.Err = err return metric } // ReadThrottleMetrics returns a metric for the given probe. Either by explicit query // or via SHOW REPLICA STATUS -func ReadThrottleMetrics(ctx context.Context, probe *Probe, metricsFunc func(context.Context) MySQLThrottleMetrics) MySQLThrottleMetrics { - if metrics := getCachedMySQLThrottleMetrics(probe); metrics != nil { +func ReadThrottleMetrics(ctx context.Context, probe *Probe, metricsFunc func(context.Context) ThrottleMetrics) ThrottleMetrics { + if metrics := getCachedThrottleMetrics(probe); metrics != nil { return metrics } started := time.Now() - mySQLThrottleMetrics := metricsFunc(ctx) + throttleMetrics := metricsFunc(ctx) - go func(metrics MySQLThrottleMetrics, started time.Time) { + go func(metrics ThrottleMetrics, started time.Time) { stats.GetOrNewGauge("ThrottlerProbesLatency", "probes latency").Set(time.Since(started).Nanoseconds()) stats.GetOrNewCounter("ThrottlerProbesTotal", "total probes").Add(1) for _, metric := range metrics { @@ -160,7 +160,7 @@ func ReadThrottleMetrics(ctx context.Context, probe *Probe, metricsFunc func(con break } } - }(mySQLThrottleMetrics, started) + }(throttleMetrics, started) - return cacheMySQLThrottleMetric(probe, mySQLThrottleMetrics) + return cacheThrottleMetric(probe, throttleMetrics) } diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 9a673bbfe54..ab19e119aa9 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -180,10 +180,10 @@ type Throttler struct { throttleTabletTypesMap map[topodatapb.TabletType]bool - mysqlThrottleMetricChan chan *mysql.MySQLThrottleMetric - mysqlClusterProbesChan chan *mysql.ClusterProbes - throttlerConfigChan chan *topodatapb.ThrottlerConfig - serialFuncChan chan func() // Used by unit tests to inject non-racy behavior + throttleMetricChan chan *mysql.ThrottleMetric + mysqlClusterProbesChan chan *mysql.ClusterProbes + throttlerConfigChan chan *topodatapb.ThrottlerConfig + serialFuncChan chan func() // Used by unit tests to inject non-racy behavior mysqlInventory *mysql.Inventory @@ -205,7 +205,7 @@ type Throttler struct { cancelEnableContext context.CancelFunc throttledAppsMutex sync.Mutex - readSelfThrottleMetrics func(context.Context) mysql.MySQLThrottleMetrics // overwritten by unit test + readSelfThrottleMetrics func(context.Context) mysql.ThrottleMetrics // overwritten by unit test httpClient *http.Client @@ -251,7 +251,7 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv }), } - throttler.mysqlThrottleMetricChan = make(chan *mysql.MySQLThrottleMetric) + throttler.throttleMetricChan = make(chan *mysql.ThrottleMetric) throttler.mysqlClusterProbesChan = make(chan *mysql.ClusterProbes) throttler.throttlerConfigChan = make(chan *topodatapb.ThrottlerConfig) throttler.serialFuncChan = make(chan func()) @@ -282,7 +282,7 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv } throttler.StoreMetricsThreshold(defaultThresholds[base.LagMetricName]) - throttler.readSelfThrottleMetrics = func(ctx context.Context) mysql.MySQLThrottleMetrics { + throttler.readSelfThrottleMetrics = func(ctx context.Context) mysql.ThrottleMetrics { return throttler.readSelfThrottleMetricsInternal(ctx) } @@ -727,8 +727,8 @@ func (throttler *Throttler) stimulatePrimaryThrottler(ctx context.Context, tmCli return nil } -func (throttler *Throttler) readSelfLoadAvgPerCore(ctx context.Context) *mysql.MySQLThrottleMetric { - metric := &mysql.MySQLThrottleMetric{ +func (throttler *Throttler) readSelfLoadAvgPerCore(ctx context.Context) *mysql.ThrottleMetric { + metric := &mysql.ThrottleMetric{ Scope: base.SelfScope, Alias: throttler.tabletAlias, } @@ -779,9 +779,9 @@ func (throttler *Throttler) readSelfLoadAvgPerCore(ctx context.Context) *mysql.M return metric } -// readSelfMySQLThrottleMetric reads the mysql metric from thi very tablet's backend mysql. -func (throttler *Throttler) readSelfMySQLThrottleMetric(ctx context.Context, query string) *mysql.MySQLThrottleMetric { - metric := &mysql.MySQLThrottleMetric{ +// readSelfThrottleMetric reads the mysql metric from thi very tablet's backend mysql. +func (throttler *Throttler) readSelfThrottleMetric(ctx context.Context, query string) *mysql.ThrottleMetric { + metric := &mysql.ThrottleMetric{ Scope: base.SelfScope, Alias: throttler.tabletAlias, } @@ -800,7 +800,7 @@ func (throttler *Throttler) readSelfMySQLThrottleMetric(ctx context.Context, que } row := tm.Named().Row() if row == nil { - return metric.WithError(fmt.Errorf("no results for readSelfMySQLThrottleMetric")) + return metric.WithError(fmt.Errorf("no results for readSelfThrottleMetric")) } metricsQueryType := mysql.GetMetricsQueryType(query) @@ -963,7 +963,7 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { throttler.collectShardMySQLMetrics(ctx, tmClient) } } - case metric := <-throttler.mysqlThrottleMetricChan: + case metric := <-throttler.throttleMetricChan: // incoming MySQL metric, frequent, as result of collectMySQLMetrics() metricResultsMap, ok := throttler.mysqlInventory.TabletMetrics[metric.GetTabletAlias()] if !ok { @@ -997,11 +997,11 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { }() } -func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClient tmclient.TabletManagerClient, probe *mysql.Probe) (probeFunc func(context.Context) mysql.MySQLThrottleMetrics) { - metricsWithError := func(err error) mysql.MySQLThrottleMetrics { - metrics := mysql.MySQLThrottleMetrics{} +func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClient tmclient.TabletManagerClient, probe *mysql.Probe) (probeFunc func(context.Context) mysql.ThrottleMetrics) { + metricsWithError := func(err error) mysql.ThrottleMetrics { + metrics := mysql.ThrottleMetrics{} for _, metricName := range base.KnownMetricNames { - metrics[metricName] = &mysql.MySQLThrottleMetric{ + metrics[metricName] = &mysql.ThrottleMetric{ Name: metricName, Scope: scope, Alias: probe.Alias, @@ -1010,30 +1010,30 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie } return metrics } - return func(ctx context.Context) mysql.MySQLThrottleMetrics { + return func(ctx context.Context) mysql.ThrottleMetrics { // Some reasonable timeout, to ensure we release connections even if they're hanging (otherwise grpc-go keeps polling those connections forever) ctx, cancel := context.WithTimeout(ctx, 4*activeCollectInterval) defer cancel() - // Hit a tablet's `check-self` via HTTP, and convert its CheckResult JSON output into a MySQLThrottleMetric - mySQLThrottleMetric := mysql.NewMySQLThrottleMetric() - mySQLThrottleMetric.Name = base.DefaultMetricName - mySQLThrottleMetric.Scope = scope - mySQLThrottleMetric.Alias = probe.Alias + // Hit a tablet's `check-self` via HTTP, and convert its CheckResult JSON output into a ThrottleMetric + throttleMetric := mysql.NewThrottleMetric() + throttleMetric.Name = base.DefaultMetricName + throttleMetric.Scope = scope + throttleMetric.Alias = probe.Alias if probe.Tablet == nil { return metricsWithError(fmt.Errorf("found nil tablet reference for alias '%v'", probe.Alias)) } - metrics := make(mysql.MySQLThrottleMetrics) + metrics := make(mysql.ThrottleMetrics) req := &tabletmanagerdatapb.CheckThrottlerRequest{MultiMetricsEnabled: true} // We leave AppName empty; it will default to VitessName anyway, and we can save some proto space resp, gRPCErr := tmClient.CheckThrottler(ctx, probe.Tablet, req) if gRPCErr != nil { return metricsWithError(fmt.Errorf("gRPC error accessing tablet %v. Err=%v", probe.Alias, gRPCErr)) } - mySQLThrottleMetric.Value = resp.Value + throttleMetric.Value = resp.Value if resp.StatusCode == http.StatusInternalServerError { - mySQLThrottleMetric.Err = fmt.Errorf("Status code: %d", resp.StatusCode) + throttleMetric.Err = fmt.Errorf("Status code: %d", resp.StatusCode) } if resp.RecentlyChecked { // We have just probed a tablet, and it reported back that someone just recently "check"ed it. @@ -1043,7 +1043,7 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie } for name, metric := range resp.Metrics { metricName := base.MetricName(name) - metrics[metricName] = &mysql.MySQLThrottleMetric{ + metrics[metricName] = &mysql.ThrottleMetric{ Name: metricName, Scope: scope, Alias: probe.Alias, @@ -1055,28 +1055,28 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie } if len(resp.Metrics) == 0 { // Backwards compatibility to v20. v20 does not report multi metrics. - mySQLThrottleMetric.Name = throttler.metricNameUsedAsDefault() - metrics[mySQLThrottleMetric.Name] = mySQLThrottleMetric + throttleMetric.Name = throttler.metricNameUsedAsDefault() + metrics[throttleMetric.Name] = throttleMetric } return metrics } } -func (throttler *Throttler) readSelfThrottleMetricsInternal(ctx context.Context) mysql.MySQLThrottleMetrics { +func (throttler *Throttler) readSelfThrottleMetricsInternal(ctx context.Context) mysql.ThrottleMetrics { - writeMetric := func(metricName base.MetricName, metric *mysql.MySQLThrottleMetric) { + writeMetric := func(metricName base.MetricName, metric *mysql.ThrottleMetric) { metric.Name = metricName select { case <-ctx.Done(): return - case throttler.mysqlThrottleMetricChan <- metric: + case throttler.throttleMetricChan <- metric: } } - go writeMetric(base.LagMetricName, throttler.readSelfMySQLThrottleMetric(ctx, sqlparser.BuildParsedQuery(defaultReplicationLagQuery, sidecar.GetIdentifier()).Query)) - go writeMetric(base.ThreadsRunningMetricName, throttler.readSelfMySQLThrottleMetric(ctx, threadsRunningQuery)) - go writeMetric(base.CustomMetricName, throttler.readSelfMySQLThrottleMetric(ctx, throttler.GetCustomMetricsQuery())) + go writeMetric(base.LagMetricName, throttler.readSelfThrottleMetric(ctx, sqlparser.BuildParsedQuery(defaultReplicationLagQuery, sidecar.GetIdentifier()).Query)) + go writeMetric(base.ThreadsRunningMetricName, throttler.readSelfThrottleMetric(ctx, threadsRunningQuery)) + go writeMetric(base.CustomMetricName, throttler.readSelfThrottleMetric(ctx, throttler.GetCustomMetricsQuery())) go writeMetric(base.LoadAvgMetricName, throttler.readSelfLoadAvgPerCore(ctx)) return nil } @@ -1124,7 +1124,7 @@ func (throttler *Throttler) collectShardMySQLMetrics(ctx context.Context, tmClie select { case <-ctx.Done(): return - case throttler.mysqlThrottleMetricChan <- metric: + case throttler.throttleMetricChan <- metric: } } }(probe) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index f2151e3b50f..367bc9761ac 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -47,26 +47,26 @@ import ( ) var ( - selfMetrics = mysql.MySQLThrottleMetrics{ - base.LagMetricName: &mysql.MySQLThrottleMetric{ + selfMetrics = mysql.ThrottleMetrics{ + base.LagMetricName: &mysql.ThrottleMetric{ Scope: base.SelfScope, Alias: "", Value: 0.3, Err: nil, }, - base.ThreadsRunningMetricName: &mysql.MySQLThrottleMetric{ + base.ThreadsRunningMetricName: &mysql.ThrottleMetric{ Scope: base.SelfScope, Alias: "", Value: 26, Err: nil, }, - base.CustomMetricName: &mysql.MySQLThrottleMetric{ + base.CustomMetricName: &mysql.ThrottleMetric{ Scope: base.SelfScope, Alias: "", Value: 17, Err: nil, }, - base.LoadAvgMetricName: &mysql.MySQLThrottleMetric{ + base.LoadAvgMetricName: &mysql.ThrottleMetric{ Scope: base.SelfScope, Alias: "", Value: 2.718, @@ -246,7 +246,7 @@ func newTestThrottler() *Throttler { throttler.MetricsThreshold.Store(math.Float64bits(0.75)) throttler.configSettings = config.NewConfigurationSettings() throttler.initConfig() - throttler.mysqlThrottleMetricChan = make(chan *mysql.MySQLThrottleMetric) + throttler.throttleMetricChan = make(chan *mysql.ThrottleMetric) throttler.mysqlClusterProbesChan = make(chan *mysql.ClusterProbes) throttler.throttlerConfigChan = make(chan *topodatapb.ThrottlerConfig) throttler.serialFuncChan = make(chan func()) @@ -272,12 +272,12 @@ func newTestThrottler() *Throttler { throttler.recentCheckDormantDiff = int64(throttler.dormantPeriod / recentCheckRateLimiterInterval) throttler.recentCheckDiff = int64(3 * time.Second / recentCheckRateLimiterInterval) - throttler.readSelfThrottleMetrics = func(ctx context.Context) mysql.MySQLThrottleMetrics { + throttler.readSelfThrottleMetrics = func(ctx context.Context) mysql.ThrottleMetrics { for _, metric := range selfMetrics { go func() { select { case <-ctx.Done(): - case throttler.mysqlThrottleMetricChan <- metric: + case throttler.throttleMetricChan <- metric: } }() } From c9ac150a9d74f46c191d625149aa803b7b880687 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:18:59 +0300 Subject: [PATCH 07/36] renamed files Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../throttle/mysql/{mysql_inventory.go => inventory.go} | 0 .../throttle/mysql/{mysql_inventory_test.go => inventory_test.go} | 0 .../mysql/{mysql_throttle_metric.go => throttle_metric.go} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename go/vt/vttablet/tabletserver/throttle/mysql/{mysql_inventory.go => inventory.go} (100%) rename go/vt/vttablet/tabletserver/throttle/mysql/{mysql_inventory_test.go => inventory_test.go} (100%) rename go/vt/vttablet/tabletserver/throttle/mysql/{mysql_throttle_metric.go => throttle_metric.go} (100%) diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/mysql_inventory.go b/go/vt/vttablet/tabletserver/throttle/mysql/inventory.go similarity index 100% rename from go/vt/vttablet/tabletserver/throttle/mysql/mysql_inventory.go rename to go/vt/vttablet/tabletserver/throttle/mysql/inventory.go diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/mysql_inventory_test.go b/go/vt/vttablet/tabletserver/throttle/mysql/inventory_test.go similarity index 100% rename from go/vt/vttablet/tabletserver/throttle/mysql/mysql_inventory_test.go rename to go/vt/vttablet/tabletserver/throttle/mysql/inventory_test.go diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go b/go/vt/vttablet/tabletserver/throttle/mysql/throttle_metric.go similarity index 100% rename from go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go rename to go/vt/vttablet/tabletserver/throttle/mysql/throttle_metric.go From dd3868f3feb9eef2a827b9893b72330d99c0e0af Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:02:55 +0300 Subject: [PATCH 08/36] rename file Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../mysql/{throttle_metric.go => throttle_metric_cache.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename go/vt/vttablet/tabletserver/throttle/mysql/{throttle_metric.go => throttle_metric_cache.go} (100%) diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/throttle_metric.go b/go/vt/vttablet/tabletserver/throttle/mysql/throttle_metric_cache.go similarity index 100% rename from go/vt/vttablet/tabletserver/throttle/mysql/throttle_metric.go rename to go/vt/vttablet/tabletserver/throttle/mysql/throttle_metric_cache.go From 2886bf3a83ddf5ab0c8deb535d01ab8709ac19bc Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:16:21 +0300 Subject: [PATCH 09/36] eliminatye throttle/mysql package; move everything under throttle/base Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../throttle/{mysql => base}/inventory.go | 8 +-- .../{mysql => base}/inventory_test.go | 10 ++- .../throttle/{mysql => base}/probe.go | 2 +- .../throttle/{mysql => base}/probe_test.go | 2 +- .../{mysql => base}/throttle_metric_cache.go | 9 ++- go/vt/vttablet/tabletserver/throttle/mysql.go | 3 +- .../tabletserver/throttle/mysql_test.go | 7 +- .../tabletserver/throttle/throttler.go | 71 +++++++++---------- .../tabletserver/throttle/throttler_test.go | 29 ++++---- 9 files changed, 65 insertions(+), 76 deletions(-) rename go/vt/vttablet/tabletserver/throttle/{mysql => base}/inventory.go (94%) rename go/vt/vttablet/tabletserver/throttle/{mysql => base}/inventory_test.go (86%) rename go/vt/vttablet/tabletserver/throttle/{mysql => base}/probe.go (99%) rename go/vt/vttablet/tabletserver/throttle/{mysql => base}/probe_test.go (99%) rename go/vt/vttablet/tabletserver/throttle/{mysql => base}/throttle_metric_cache.go (96%) diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/inventory.go b/go/vt/vttablet/tabletserver/throttle/base/inventory.go similarity index 94% rename from go/vt/vttablet/tabletserver/throttle/mysql/inventory.go rename to go/vt/vttablet/tabletserver/throttle/base/inventory.go index f8ae0e26f11..d4776bf19b2 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/inventory.go +++ b/go/vt/vttablet/tabletserver/throttle/base/inventory.go @@ -39,14 +39,10 @@ limitations under the License. SOFTWARE. */ -package mysql - -import ( - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" -) +package base // TabletResultMap maps a tablet to a result -type TabletResultMap map[string]base.MetricResultMap +type TabletResultMap map[string]MetricResultMap func (m TabletResultMap) Split(alias string) (withAlias TabletResultMap, all TabletResultMap) { withAlias = make(TabletResultMap) diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/inventory_test.go b/go/vt/vttablet/tabletserver/throttle/base/inventory_test.go similarity index 86% rename from go/vt/vttablet/tabletserver/throttle/mysql/inventory_test.go rename to go/vt/vttablet/tabletserver/throttle/base/inventory_test.go index fe6204a36c2..2850a3b3035 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/inventory_test.go +++ b/go/vt/vttablet/tabletserver/throttle/base/inventory_test.go @@ -14,22 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mysql +package base import ( "testing" "github.com/stretchr/testify/assert" "golang.org/x/exp/maps" - - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" ) func TestTabletResultMapSplit(t *testing.T) { tabletResultMap := TabletResultMap{ - "a": make(base.MetricResultMap), - "b": make(base.MetricResultMap), - "c": make(base.MetricResultMap), + "a": make(MetricResultMap), + "b": make(MetricResultMap), + "c": make(MetricResultMap), } { withAlias, all := tabletResultMap.Split("b") diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/probe.go b/go/vt/vttablet/tabletserver/throttle/base/probe.go similarity index 99% rename from go/vt/vttablet/tabletserver/throttle/mysql/probe.go rename to go/vt/vttablet/tabletserver/throttle/base/probe.go index af0d660096e..0fe813d571e 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/probe.go +++ b/go/vt/vttablet/tabletserver/throttle/base/probe.go @@ -39,7 +39,7 @@ limitations under the License. SOFTWARE. */ -package mysql +package base import ( "fmt" diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/probe_test.go b/go/vt/vttablet/tabletserver/throttle/base/probe_test.go similarity index 99% rename from go/vt/vttablet/tabletserver/throttle/mysql/probe_test.go rename to go/vt/vttablet/tabletserver/throttle/base/probe_test.go index 8f489f39258..af1b09ec181 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/probe_test.go +++ b/go/vt/vttablet/tabletserver/throttle/base/probe_test.go @@ -39,7 +39,7 @@ limitations under the License. SOFTWARE. */ -package mysql +package base import ( "testing" diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/throttle_metric_cache.go b/go/vt/vttablet/tabletserver/throttle/base/throttle_metric_cache.go similarity index 96% rename from go/vt/vttablet/tabletserver/throttle/mysql/throttle_metric_cache.go rename to go/vt/vttablet/tabletserver/throttle/base/throttle_metric_cache.go index d4bd079572a..8695cb83229 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/throttle_metric_cache.go +++ b/go/vt/vttablet/tabletserver/throttle/base/throttle_metric_cache.go @@ -39,7 +39,7 @@ limitations under the License. SOFTWARE. */ -package mysql +package base import ( "context" @@ -49,7 +49,6 @@ import ( "github.com/patrickmn/go-cache" "vitess.io/vitess/go/stats" - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" ) // MetricsQueryType indicates the type of metrics query on MySQL backend. See following. @@ -111,14 +110,14 @@ func GetMetricsQueryType(query string) MetricsQueryType { // ThrottleMetric has the probed metric for a tablet type ThrottleMetric struct { // nolint:revive - Name base.MetricName - Scope base.Scope + Name MetricName + Scope Scope Alias string Value float64 Err error } -type ThrottleMetrics map[base.MetricName]*ThrottleMetric // nolint:revive +type ThrottleMetrics map[MetricName]*ThrottleMetric // nolint:revive // NewThrottleMetric creates a new ThrottleMetric func NewThrottleMetric() *ThrottleMetric { diff --git a/go/vt/vttablet/tabletserver/throttle/mysql.go b/go/vt/vttablet/tabletserver/throttle/mysql.go index ee24bf38b40..c952c644638 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql.go @@ -46,13 +46,12 @@ import ( "sort" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/mysql" ) func aggregateMetricResults( ctx context.Context, metricName base.MetricName, - tabletResultsMap mysql.TabletResultMap, + tabletResultsMap base.TabletResultMap, ignoreHostsCount int, IgnoreDialTCPErrors bool, ignoreHostsThreshold float64, diff --git a/go/vt/vttablet/tabletserver/throttle/mysql_test.go b/go/vt/vttablet/tabletserver/throttle/mysql_test.go index 8fb25f0929d..ade8a6e6e0f 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql_test.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql_test.go @@ -46,7 +46,6 @@ import ( "testing" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/mysql" "github.com/stretchr/testify/assert" ) @@ -80,7 +79,7 @@ func noSuchMetricMap() base.MetricResultMap { func TestAggregateMetricResultsNoErrors(t *testing.T) { ctx := context.Background() - tabletResultsMap := mysql.TabletResultMap{ + tabletResultsMap := base.TabletResultMap{ alias1: newMetricResultMap(1.2), alias2: newMetricResultMap(1.7), alias3: newMetricResultMap(0.3), @@ -128,7 +127,7 @@ func TestAggregateMetricResultsNoErrors(t *testing.T) { func TestAggregateMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { ctx := context.Background() - tabletResultsMap := mysql.TabletResultMap{ + tabletResultsMap := base.TabletResultMap{ alias1: newMetricResultMap(1.2), alias2: newMetricResultMap(1.7), alias3: newMetricResultMap(0.3), @@ -176,7 +175,7 @@ func TestAggregateMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { func TestAggregateMetricResultsWithErrors(t *testing.T) { ctx := context.Background() - tabletResultsMap := mysql.TabletResultMap{ + tabletResultsMap := base.TabletResultMap{ alias1: newMetricResultMap(1.2), alias2: newMetricResultMap(1.7), alias3: newMetricResultMap(0.3), diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index ab19e119aa9..6055d0fa487 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -78,7 +78,6 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/config" - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/mysql" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/throttlerapp" "vitess.io/vitess/go/vt/vttablet/tmclient" ) @@ -180,12 +179,12 @@ type Throttler struct { throttleTabletTypesMap map[topodatapb.TabletType]bool - throttleMetricChan chan *mysql.ThrottleMetric - mysqlClusterProbesChan chan *mysql.ClusterProbes + throttleMetricChan chan *base.ThrottleMetric + mysqlClusterProbesChan chan *base.ClusterProbes throttlerConfigChan chan *topodatapb.ThrottlerConfig serialFuncChan chan func() // Used by unit tests to inject non-racy behavior - mysqlInventory *mysql.Inventory + mysqlInventory *base.Inventory metricsQuery atomic.Value customMetricsQuery atomic.Value @@ -205,7 +204,7 @@ type Throttler struct { cancelEnableContext context.CancelFunc throttledAppsMutex sync.Mutex - readSelfThrottleMetrics func(context.Context) mysql.ThrottleMetrics // overwritten by unit test + readSelfThrottleMetrics func(context.Context) base.ThrottleMetrics // overwritten by unit test httpClient *http.Client @@ -251,11 +250,11 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv }), } - throttler.throttleMetricChan = make(chan *mysql.ThrottleMetric) - throttler.mysqlClusterProbesChan = make(chan *mysql.ClusterProbes) + throttler.throttleMetricChan = make(chan *base.ThrottleMetric) + throttler.mysqlClusterProbesChan = make(chan *base.ClusterProbes) throttler.throttlerConfigChan = make(chan *topodatapb.ThrottlerConfig) throttler.serialFuncChan = make(chan func()) - throttler.mysqlInventory = mysql.NewInventory() + throttler.mysqlInventory = base.NewInventory() throttler.throttledApps = cache.New(cache.NoExpiration, 0) throttler.metricThresholds = cache.New(cache.NoExpiration, 0) @@ -282,7 +281,7 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv } throttler.StoreMetricsThreshold(defaultThresholds[base.LagMetricName]) - throttler.readSelfThrottleMetrics = func(ctx context.Context) mysql.ThrottleMetrics { + throttler.readSelfThrottleMetrics = func(ctx context.Context) base.ThrottleMetrics { return throttler.readSelfThrottleMetricsInternal(ctx) } @@ -727,8 +726,8 @@ func (throttler *Throttler) stimulatePrimaryThrottler(ctx context.Context, tmCli return nil } -func (throttler *Throttler) readSelfLoadAvgPerCore(ctx context.Context) *mysql.ThrottleMetric { - metric := &mysql.ThrottleMetric{ +func (throttler *Throttler) readSelfLoadAvgPerCore(ctx context.Context) *base.ThrottleMetric { + metric := &base.ThrottleMetric{ Scope: base.SelfScope, Alias: throttler.tabletAlias, } @@ -780,8 +779,8 @@ func (throttler *Throttler) readSelfLoadAvgPerCore(ctx context.Context) *mysql.T } // readSelfThrottleMetric reads the mysql metric from thi very tablet's backend mysql. -func (throttler *Throttler) readSelfThrottleMetric(ctx context.Context, query string) *mysql.ThrottleMetric { - metric := &mysql.ThrottleMetric{ +func (throttler *Throttler) readSelfThrottleMetric(ctx context.Context, query string) *base.ThrottleMetric { + metric := &base.ThrottleMetric{ Scope: base.SelfScope, Alias: throttler.tabletAlias, } @@ -803,15 +802,15 @@ func (throttler *Throttler) readSelfThrottleMetric(ctx context.Context, query st return metric.WithError(fmt.Errorf("no results for readSelfThrottleMetric")) } - metricsQueryType := mysql.GetMetricsQueryType(query) + metricsQueryType := base.GetMetricsQueryType(query) switch metricsQueryType { - case mysql.MetricsQueryTypeSelect: + case base.MetricsQueryTypeSelect: // We expect a single row, single column result. // The "for" iteration below is just a way to get first result without knowing column name for k := range row { metric.Value, metric.Err = row.ToFloat64(k) } - case mysql.MetricsQueryTypeShowGlobal: + case base.MetricsQueryTypeShowGlobal: metric.Value, metric.Err = strconv.ParseFloat(row["Value"].ToString(), 64) default: metric.Err = fmt.Errorf("Unsupported metrics query type for query: %s", throttler.GetMetricsQuery()) @@ -997,11 +996,11 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { }() } -func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClient tmclient.TabletManagerClient, probe *mysql.Probe) (probeFunc func(context.Context) mysql.ThrottleMetrics) { - metricsWithError := func(err error) mysql.ThrottleMetrics { - metrics := mysql.ThrottleMetrics{} +func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClient tmclient.TabletManagerClient, probe *base.Probe) (probeFunc func(context.Context) base.ThrottleMetrics) { + metricsWithError := func(err error) base.ThrottleMetrics { + metrics := base.ThrottleMetrics{} for _, metricName := range base.KnownMetricNames { - metrics[metricName] = &mysql.ThrottleMetric{ + metrics[metricName] = &base.ThrottleMetric{ Name: metricName, Scope: scope, Alias: probe.Alias, @@ -1010,13 +1009,13 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie } return metrics } - return func(ctx context.Context) mysql.ThrottleMetrics { + return func(ctx context.Context) base.ThrottleMetrics { // Some reasonable timeout, to ensure we release connections even if they're hanging (otherwise grpc-go keeps polling those connections forever) ctx, cancel := context.WithTimeout(ctx, 4*activeCollectInterval) defer cancel() // Hit a tablet's `check-self` via HTTP, and convert its CheckResult JSON output into a ThrottleMetric - throttleMetric := mysql.NewThrottleMetric() + throttleMetric := base.NewThrottleMetric() throttleMetric.Name = base.DefaultMetricName throttleMetric.Scope = scope throttleMetric.Alias = probe.Alias @@ -1024,7 +1023,7 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie if probe.Tablet == nil { return metricsWithError(fmt.Errorf("found nil tablet reference for alias '%v'", probe.Alias)) } - metrics := make(mysql.ThrottleMetrics) + metrics := make(base.ThrottleMetrics) req := &tabletmanagerdatapb.CheckThrottlerRequest{MultiMetricsEnabled: true} // We leave AppName empty; it will default to VitessName anyway, and we can save some proto space resp, gRPCErr := tmClient.CheckThrottler(ctx, probe.Tablet, req) @@ -1043,7 +1042,7 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie } for name, metric := range resp.Metrics { metricName := base.MetricName(name) - metrics[metricName] = &mysql.ThrottleMetric{ + metrics[metricName] = &base.ThrottleMetric{ Name: metricName, Scope: scope, Alias: probe.Alias, @@ -1063,9 +1062,9 @@ func (throttler *Throttler) generateTabletProbeFunction(scope base.Scope, tmClie } } -func (throttler *Throttler) readSelfThrottleMetricsInternal(ctx context.Context) mysql.ThrottleMetrics { +func (throttler *Throttler) readSelfThrottleMetricsInternal(ctx context.Context) base.ThrottleMetrics { - writeMetric := func(metricName base.MetricName, metric *mysql.ThrottleMetric) { + writeMetric := func(metricName base.MetricName, metric *base.ThrottleMetric) { metric.Name = metricName select { case <-ctx.Done(): @@ -1096,7 +1095,7 @@ func (throttler *Throttler) collectSelfMySQLMetrics(ctx context.Context, tmClien defer atomic.StoreInt64(&probe.QueryInProgress, 0) // Throttler is probing its own tablet's metrics: - _ = mysql.ReadThrottleMetrics(ctx, probe, throttler.readSelfThrottleMetrics) + _ = base.ReadThrottleMetrics(ctx, probe, throttler.readSelfThrottleMetrics) }() } @@ -1108,7 +1107,7 @@ func (throttler *Throttler) collectShardMySQLMetrics(ctx context.Context, tmClie // We skip collecting our own metrics continue } - go func(probe *mysql.Probe) { + go func(probe *base.Probe) { // Avoid querying the same server twice at the same time. If previous read is still there, // we avoid re-reading it. if !atomic.CompareAndSwapInt64(&probe.QueryInProgress, 0, 1) { @@ -1119,7 +1118,7 @@ func (throttler *Throttler) collectShardMySQLMetrics(ctx context.Context, tmClie // Throttler probing other tablets: throttleMetricFunc := throttler.generateTabletProbeFunction(base.ShardScope, tmClient, probe) - throttleMetrics := mysql.ReadThrottleMetrics(ctx, probe, throttleMetricFunc) + throttleMetrics := base.ReadThrottleMetrics(ctx, probe, throttleMetricFunc) for _, metric := range throttleMetrics { select { case <-ctx.Done(): @@ -1134,7 +1133,7 @@ func (throttler *Throttler) collectShardMySQLMetrics(ctx context.Context, tmClie // refreshMySQLInventory will re-structure the inventory based on reading config settings func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { // distribute the query/threshold from the throttler down to the cluster settings and from there to the probes - addProbe := func(alias string, tablet *topodatapb.Tablet, scope base.Scope, mysqlSettings *config.MySQLConfigurationSettings, probes mysql.Probes) bool { + addProbe := func(alias string, tablet *topodatapb.Tablet, scope base.Scope, mysqlSettings *config.MySQLConfigurationSettings, probes base.Probes) bool { for _, ignore := range mysqlSettings.IgnoreHosts { if strings.Contains(alias, ignore) { log.Infof("Throttler: tablet ignored: %+v", alias) @@ -1152,7 +1151,7 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { } } - probe := &mysql.Probe{ + probe := &base.Probe{ Alias: alias, Tablet: tablet, CacheMillis: mysqlSettings.CacheMillis, @@ -1161,7 +1160,7 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { return true } - attemptWriteProbes := func(clusterProbes *mysql.ClusterProbes) error { + attemptWriteProbes := func(clusterProbes *base.ClusterProbes) error { select { case <-ctx.Done(): return ctx.Err() @@ -1188,9 +1187,9 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { // config may dynamically change, but internal structure (config.Settings().MySQLStore.Clusters in our case) // is immutable and can only be _replaced_. Hence, it's safe to read in a goroutine: collect := func() error { - clusterProbes := &mysql.ClusterProbes{ + clusterProbes := &base.ClusterProbes{ IgnoreHostsCount: clusterSettingsCopy.IgnoreHostsCount, - TabletProbes: mysql.NewProbes(), + TabletProbes: base.NewProbes(), } // self tablet addProbe(throttler.tabletAlias, nil, base.SelfScope, &clusterSettingsCopy, clusterProbes.TabletProbes) @@ -1233,7 +1232,7 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { } // synchronous update of inventory -func (throttler *Throttler) updateMySQLClusterProbes(ctx context.Context, clusterProbes *mysql.ClusterProbes) error { +func (throttler *Throttler) updateMySQLClusterProbes(ctx context.Context, clusterProbes *base.ClusterProbes) error { throttler.mysqlInventory.ClustersProbes = clusterProbes.TabletProbes throttler.mysqlInventory.IgnoreHostsCount = clusterProbes.IgnoreHostsCount throttler.mysqlInventory.IgnoreHostsThreshold = clusterProbes.IgnoreHostsThreshold @@ -1250,7 +1249,7 @@ func (throttler *Throttler) metricNameUsedAsDefault() base.MetricName { // synchronous aggregation of collected data func (throttler *Throttler) aggregateMySQLMetrics(ctx context.Context) error { metricNameUsedAsDefault := throttler.metricNameUsedAsDefault() - aggregateTabletsMetrics := func(scope base.Scope, metricName base.MetricName, tabletResultsMap mysql.TabletResultMap) { + aggregateTabletsMetrics := func(scope base.Scope, metricName base.MetricName, tabletResultsMap base.TabletResultMap) { ignoreHostsCount := throttler.mysqlInventory.IgnoreHostsCount ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold ignoreDialTCPErrors := throttler.configSettings.MySQLStore.IgnoreDialTCPErrors diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index 367bc9761ac..c9025ca7e1f 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -38,7 +38,6 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/config" - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/mysql" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/throttlerapp" "vitess.io/vitess/go/vt/vttablet/tmclient" @@ -47,26 +46,26 @@ import ( ) var ( - selfMetrics = mysql.ThrottleMetrics{ - base.LagMetricName: &mysql.ThrottleMetric{ + selfMetrics = base.ThrottleMetrics{ + base.LagMetricName: &base.ThrottleMetric{ Scope: base.SelfScope, Alias: "", Value: 0.3, Err: nil, }, - base.ThreadsRunningMetricName: &mysql.ThrottleMetric{ + base.ThreadsRunningMetricName: &base.ThrottleMetric{ Scope: base.SelfScope, Alias: "", Value: 26, Err: nil, }, - base.CustomMetricName: &mysql.ThrottleMetric{ + base.CustomMetricName: &base.ThrottleMetric{ Scope: base.SelfScope, Alias: "", Value: 17, Err: nil, }, - base.LoadAvgMetricName: &mysql.ThrottleMetric{ + base.LoadAvgMetricName: &base.ThrottleMetric{ Scope: base.SelfScope, Alias: "", Value: 2.718, @@ -234,10 +233,10 @@ func newTestThrottler() *Throttler { env := tabletenv.NewEnv(vtenv.NewTestEnv(), nil, "TabletServerTest") throttler := &Throttler{ - mysqlClusterProbesChan: make(chan *mysql.ClusterProbes), + mysqlClusterProbesChan: make(chan *base.ClusterProbes), heartbeatWriter: &FakeHeartbeatWriter{}, ts: &FakeTopoServer{}, - mysqlInventory: mysql.NewInventory(), + mysqlInventory: base.NewInventory(), pool: connpool.NewPool(env, "ThrottlerPool", tabletenv.ConnPoolConfig{}), tabletTypeFunc: func() topodatapb.TabletType { return topodatapb.TabletType_PRIMARY }, overrideTmClient: &fakeTMClient{}, @@ -246,11 +245,11 @@ func newTestThrottler() *Throttler { throttler.MetricsThreshold.Store(math.Float64bits(0.75)) throttler.configSettings = config.NewConfigurationSettings() throttler.initConfig() - throttler.throttleMetricChan = make(chan *mysql.ThrottleMetric) - throttler.mysqlClusterProbesChan = make(chan *mysql.ClusterProbes) + throttler.throttleMetricChan = make(chan *base.ThrottleMetric) + throttler.mysqlClusterProbesChan = make(chan *base.ClusterProbes) throttler.throttlerConfigChan = make(chan *topodatapb.ThrottlerConfig) throttler.serialFuncChan = make(chan func()) - throttler.mysqlInventory = mysql.NewInventory() + throttler.mysqlInventory = base.NewInventory() throttler.throttledApps = cache.New(cache.NoExpiration, 0) throttler.metricThresholds = cache.New(cache.NoExpiration, 0) @@ -272,7 +271,7 @@ func newTestThrottler() *Throttler { throttler.recentCheckDormantDiff = int64(throttler.dormantPeriod / recentCheckRateLimiterInterval) throttler.recentCheckDiff = int64(3 * time.Second / recentCheckRateLimiterInterval) - throttler.readSelfThrottleMetrics = func(ctx context.Context) mysql.ThrottleMetrics { + throttler.readSelfThrottleMetrics = func(ctx context.Context) base.ThrottleMetrics { for _, metric := range selfMetrics { go func() { select { @@ -902,10 +901,10 @@ func TestRefreshMySQLInventory(t *testing.T) { configSettings := config.NewConfigurationSettings() throttler := &Throttler{ - mysqlClusterProbesChan: make(chan *mysql.ClusterProbes), + mysqlClusterProbesChan: make(chan *base.ClusterProbes), metricThresholds: cache.New(cache.NoExpiration, 0), ts: &FakeTopoServer{}, - mysqlInventory: mysql.NewInventory(), + mysqlInventory: base.NewInventory(), } throttler.metricsQuery.Store(metricsQuery) throttler.configSettings = configSettings @@ -916,7 +915,7 @@ func TestRefreshMySQLInventory(t *testing.T) { testName := fmt.Sprintf("leader=%t", throttler.isLeader.Load()) t.Run(testName, func(t *testing.T) { // validateProbesCount expects number of probes according to cluster name and throttler's leadership status - validateProbesCount := func(t *testing.T, probes mysql.Probes) { + validateProbesCount := func(t *testing.T, probes base.Probes) { if throttler.isLeader.Load() { assert.Equal(t, 3, len(probes)) } else { From f6e8a5546f2a8a2cab0a9cf5c22301130b5cf9a1 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:32:53 +0300 Subject: [PATCH 10/36] Remove unused type Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/check.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/check.go b/go/vt/vttablet/tabletserver/throttle/check.go index 1287ef945ed..069a1c275f5 100644 --- a/go/vt/vttablet/tabletserver/throttle/check.go +++ b/go/vt/vttablet/tabletserver/throttle/check.go @@ -62,8 +62,6 @@ var ( statsThrottlerCheckAnyError = stats.GetOrNewCounter("ThrottlerCheckAnyError", "total number of failed checks") ) -type ThrottlePriority int - // CheckFlags provide hints for a check type CheckFlags struct { Scope base.Scope From 1ea1b5133589caecc5aab9e1cb4cf5082fcbdbdd Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:38:31 +0300 Subject: [PATCH 11/36] remove ctx argument Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/mysql.go | 2 - .../tabletserver/throttle/mysql_test.go | 42 +++++++++---------- .../tabletserver/throttle/throttler.go | 2 +- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/mysql.go b/go/vt/vttablet/tabletserver/throttle/mysql.go index c952c644638..d28a234a087 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql.go @@ -42,14 +42,12 @@ limitations under the License. package throttle import ( - "context" "sort" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" ) func aggregateMetricResults( - ctx context.Context, metricName base.MetricName, tabletResultsMap base.TabletResultMap, ignoreHostsCount int, diff --git a/go/vt/vttablet/tabletserver/throttle/mysql_test.go b/go/vt/vttablet/tabletserver/throttle/mysql_test.go index ade8a6e6e0f..182cf6b0cea 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql_test.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql_test.go @@ -42,7 +42,6 @@ limitations under the License. package throttle import ( - "context" "testing" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" @@ -78,7 +77,6 @@ func noSuchMetricMap() base.MetricResultMap { } func TestAggregateMetricResultsNoErrors(t *testing.T) { - ctx := context.Background() tabletResultsMap := base.TabletResultMap{ alias1: newMetricResultMap(1.2), alias2: newMetricResultMap(1.7), @@ -88,37 +86,37 @@ func TestAggregateMetricResultsNoErrors(t *testing.T) { } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.2) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.1) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 3, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 3, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 4, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 4, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.3) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 5, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 5, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.3) @@ -126,7 +124,6 @@ func TestAggregateMetricResultsNoErrors(t *testing.T) { } func TestAggregateMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { - ctx := context.Background() tabletResultsMap := base.TabletResultMap{ alias1: newMetricResultMap(1.2), alias2: newMetricResultMap(1.7), @@ -136,37 +133,37 @@ func TestAggregateMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 1.0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 1.0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.2) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 1.0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.1) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 3, false, 1.0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 3, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 4, false, 1.0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 4, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 5, false, 1.0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 5, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) @@ -174,7 +171,6 @@ func TestAggregateMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { } func TestAggregateMetricResultsWithErrors(t *testing.T) { - ctx := context.Background() tabletResultsMap := base.TabletResultMap{ alias1: newMetricResultMap(1.2), alias2: newMetricResultMap(1.7), @@ -184,25 +180,25 @@ func TestAggregateMetricResultsWithErrors(t *testing.T) { } t.Run("nonexistent", func(t *testing.T) { - worstMetric := aggregateMetricResults(ctx, nonexistentMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateMetricResults(nonexistentMetricName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("no ignore", func(t *testing.T) { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("ignore 1", func(t *testing.T) { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, 1.7, value) }) t.Run("ignore 2", func(t *testing.T) { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, 1.2, value) @@ -210,19 +206,19 @@ func TestAggregateMetricResultsWithErrors(t *testing.T) { tabletResultsMap[alias1][base.DefaultMetricName] = base.NoSuchMetric t.Run("no such metric", func(t *testing.T) { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("no such metric, ignore 1", func(t *testing.T) { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 1, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("metric found", func(t *testing.T) { - worstMetric := aggregateMetricResults(ctx, base.DefaultMetricName, tabletResultsMap, 2, false, 0) + worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 6055d0fa487..9f98baec667 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1254,7 +1254,7 @@ func (throttler *Throttler) aggregateMySQLMetrics(ctx context.Context) error { ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold ignoreDialTCPErrors := throttler.configSettings.MySQLStore.IgnoreDialTCPErrors - aggregatedMetric := aggregateMetricResults(ctx, metricName, tabletResultsMap, ignoreHostsCount, ignoreDialTCPErrors, ignoreHostsThreshold) + aggregatedMetric := aggregateMetricResults(metricName, tabletResultsMap, ignoreHostsCount, ignoreDialTCPErrors, ignoreHostsThreshold) aggregatedMetricName := metricName.AggregatedName(scope) throttler.aggregatedMetrics.Set(aggregatedMetricName, aggregatedMetric, cache.DefaultExpiration) if metricName == metricNameUsedAsDefault { From 1bddcc019c35b58e88dc507116a39995ed1ddf4a Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:41:23 +0300 Subject: [PATCH 12/36] rename aggregateMetricResults -> aggregateTabletMetricResults Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/mysql.go | 2 +- .../tabletserver/throttle/mysql_test.go | 44 +++++++++---------- .../tabletserver/throttle/throttler.go | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/mysql.go b/go/vt/vttablet/tabletserver/throttle/mysql.go index d28a234a087..cc49163d52b 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql.go @@ -47,7 +47,7 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" ) -func aggregateMetricResults( +func aggregateTabletMetricResults( metricName base.MetricName, tabletResultsMap base.TabletResultMap, ignoreHostsCount int, diff --git a/go/vt/vttablet/tabletserver/throttle/mysql_test.go b/go/vt/vttablet/tabletserver/throttle/mysql_test.go index 182cf6b0cea..fe320d569be 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql_test.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql_test.go @@ -76,7 +76,7 @@ func noSuchMetricMap() base.MetricResultMap { return result } -func TestAggregateMetricResultsNoErrors(t *testing.T) { +func TestAggregateTabletMetricResultsNoErrors(t *testing.T) { tabletResultsMap := base.TabletResultMap{ alias1: newMetricResultMap(1.2), alias2: newMetricResultMap(1.7), @@ -86,44 +86,44 @@ func TestAggregateMetricResultsNoErrors(t *testing.T) { } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.2) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.1) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 3, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 3, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 4, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 4, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.3) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 5, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 5, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.3) } } -func TestAggregateMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { +func TestAggregateTabletMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { tabletResultsMap := base.TabletResultMap{ alias1: newMetricResultMap(1.2), alias2: newMetricResultMap(1.7), @@ -133,44 +133,44 @@ func TestAggregateMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 1.0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 1.0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.2) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 1.0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.1) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 3, false, 1.0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 3, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 4, false, 1.0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 4, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 5, false, 1.0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 5, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } } -func TestAggregateMetricResultsWithErrors(t *testing.T) { +func TestAggregateTabletMetricResultsWithErrors(t *testing.T) { tabletResultsMap := base.TabletResultMap{ alias1: newMetricResultMap(1.2), alias2: newMetricResultMap(1.7), @@ -180,25 +180,25 @@ func TestAggregateMetricResultsWithErrors(t *testing.T) { } t.Run("nonexistent", func(t *testing.T) { - worstMetric := aggregateMetricResults(nonexistentMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateTabletMetricResults(nonexistentMetricName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("no ignore", func(t *testing.T) { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("ignore 1", func(t *testing.T) { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, 1.7, value) }) t.Run("ignore 2", func(t *testing.T) { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, 1.2, value) @@ -206,19 +206,19 @@ func TestAggregateMetricResultsWithErrors(t *testing.T) { tabletResultsMap[alias1][base.DefaultMetricName] = base.NoSuchMetric t.Run("no such metric", func(t *testing.T) { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("no such metric, ignore 1", func(t *testing.T) { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, base.ErrNoSuchMetric, err) }) t.Run("metric found", func(t *testing.T) { - worstMetric := aggregateMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) + worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 9f98baec667..973c39dd842 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1254,7 +1254,7 @@ func (throttler *Throttler) aggregateMySQLMetrics(ctx context.Context) error { ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold ignoreDialTCPErrors := throttler.configSettings.MySQLStore.IgnoreDialTCPErrors - aggregatedMetric := aggregateMetricResults(metricName, tabletResultsMap, ignoreHostsCount, ignoreDialTCPErrors, ignoreHostsThreshold) + aggregatedMetric := aggregateTabletMetricResults(metricName, tabletResultsMap, ignoreHostsCount, ignoreDialTCPErrors, ignoreHostsThreshold) aggregatedMetricName := metricName.AggregatedName(scope) throttler.aggregatedMetrics.Set(aggregatedMetricName, aggregatedMetric, cache.DefaultExpiration) if metricName == metricNameUsedAsDefault { From ec62bbebf1aa646b04b01953452a7fa5835769d6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:46:55 +0300 Subject: [PATCH 13/36] shuffle code around, moving aggregateTabletMetricResults into 'base' package Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/throttle/base/inventory.go | 11 - go/vt/vttablet/tabletserver/throttle/mysql.go | 117 --------- .../tabletserver/throttle/mysql_test.go | 226 ------------------ .../tabletserver/throttle/throttler.go | 2 +- 4 files changed, 1 insertion(+), 355 deletions(-) delete mode 100644 go/vt/vttablet/tabletserver/throttle/mysql.go delete mode 100644 go/vt/vttablet/tabletserver/throttle/mysql_test.go diff --git a/go/vt/vttablet/tabletserver/throttle/base/inventory.go b/go/vt/vttablet/tabletserver/throttle/base/inventory.go index d4776bf19b2..5294caf1115 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/inventory.go +++ b/go/vt/vttablet/tabletserver/throttle/base/inventory.go @@ -41,17 +41,6 @@ limitations under the License. package base -// TabletResultMap maps a tablet to a result -type TabletResultMap map[string]MetricResultMap - -func (m TabletResultMap) Split(alias string) (withAlias TabletResultMap, all TabletResultMap) { - withAlias = make(TabletResultMap) - if val, ok := m[alias]; ok { - withAlias[alias] = val - } - return withAlias, m -} - // Inventory has the operational data about probes, their metrics, and relevant configuration type Inventory struct { ClustersProbes Probes diff --git a/go/vt/vttablet/tabletserver/throttle/mysql.go b/go/vt/vttablet/tabletserver/throttle/mysql.go deleted file mode 100644 index cc49163d52b..00000000000 --- a/go/vt/vttablet/tabletserver/throttle/mysql.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2023 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE -/* - MIT License - - Copyright (c) 2017 GitHub - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -package throttle - -import ( - "sort" - - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" -) - -func aggregateTabletMetricResults( - metricName base.MetricName, - tabletResultsMap base.TabletResultMap, - ignoreHostsCount int, - IgnoreDialTCPErrors bool, - ignoreHostsThreshold float64, -) (worstMetric base.MetricResult) { - // probes is known not to change. It can be *replaced*, but not changed. - // so it's safe to iterate it - probeValues := []float64{} - for _, tabletMetricResults := range tabletResultsMap { - tabletMetricResult, ok := tabletMetricResults[metricName] - if !ok { - return base.NoSuchMetric - } - if tabletMetricResult == nil { - return base.NoMetricResultYet - } - value, err := tabletMetricResult.Get() - if err != nil { - if IgnoreDialTCPErrors && base.IsDialTCPError(err) { - continue - } - if ignoreHostsCount > 0 { - // ok to skip this error - ignoreHostsCount = ignoreHostsCount - 1 - continue - } - return tabletMetricResult - } - - // No error - probeValues = append(probeValues, value) - } - if len(probeValues) == 0 { - return base.NoHostsMetricResult - } - - // If we got here, that means no errors (or good-to-skip errors) - sort.Float64s(probeValues) - // probeValues sorted ascending (from best, ie smallest, to worst, ie largest) - for ignoreHostsCount > 0 { - goodToIgnore := func() bool { - // Note that these hosts don't have errors - numProbeValues := len(probeValues) - if numProbeValues <= 1 { - // We wish to retain at least one host - return false - } - if ignoreHostsThreshold <= 0 { - // No threshold conditional (or implicitly "any value exceeds the threshold") - return true - } - if worstValue := probeValues[numProbeValues-1]; worstValue > ignoreHostsThreshold { - return true - } - return false - }() - if goodToIgnore { - probeValues = probeValues[0 : len(probeValues)-1] - } - // And, whether ignored or not, we are reducing our tokens - ignoreHostsCount = ignoreHostsCount - 1 - } - worstValue := probeValues[len(probeValues)-1] - worstMetric = base.NewSimpleMetricResult(worstValue) - return worstMetric -} diff --git a/go/vt/vttablet/tabletserver/throttle/mysql_test.go b/go/vt/vttablet/tabletserver/throttle/mysql_test.go deleted file mode 100644 index fe320d569be..00000000000 --- a/go/vt/vttablet/tabletserver/throttle/mysql_test.go +++ /dev/null @@ -1,226 +0,0 @@ -/* -Copyright 2023 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE -/* - MIT License - - Copyright (c) 2017 GitHub - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -package throttle - -import ( - "testing" - - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" - - "github.com/stretchr/testify/assert" -) - -var ( - alias1 = "zone1-0001" - alias2 = "zone1-0002" - alias3 = "zone1-0003" - alias4 = "zone1-0004" - alias5 = "zone1-0005" -) - -const ( - nonexistentMetricName base.MetricName = "nonexistent" -) - -func newMetricResultMap(val float64) base.MetricResultMap { - return base.MetricResultMap{ - base.DefaultMetricName: base.NewSimpleMetricResult(val), - base.LagMetricName: base.NewSimpleMetricResult(val), - base.LoadAvgMetricName: base.NewSimpleMetricResult(3.14), - } -} -func noSuchMetricMap() base.MetricResultMap { - result := make(base.MetricResultMap) - for _, metricName := range base.KnownMetricNames { - result[metricName] = base.NoSuchMetric - } - return result -} - -func TestAggregateTabletMetricResultsNoErrors(t *testing.T) { - tabletResultsMap := base.TabletResultMap{ - alias1: newMetricResultMap(1.2), - alias2: newMetricResultMap(1.7), - alias3: newMetricResultMap(0.3), - alias4: newMetricResultMap(0.6), - alias5: newMetricResultMap(1.1), - } - - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 1.7) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 1.2) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 1.1) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 3, false, 0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 0.6) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 4, false, 0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 0.3) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 5, false, 0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 0.3) - } -} - -func TestAggregateTabletMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { - tabletResultsMap := base.TabletResultMap{ - alias1: newMetricResultMap(1.2), - alias2: newMetricResultMap(1.7), - alias3: newMetricResultMap(0.3), - alias4: newMetricResultMap(0.6), - alias5: newMetricResultMap(1.1), - } - - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 1.0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 1.7) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 1.0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 1.2) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 1.0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 1.1) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 3, false, 1.0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 0.6) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 4, false, 1.0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 0.6) - } - { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 5, false, 1.0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 0.6) - } -} - -func TestAggregateTabletMetricResultsWithErrors(t *testing.T) { - tabletResultsMap := base.TabletResultMap{ - alias1: newMetricResultMap(1.2), - alias2: newMetricResultMap(1.7), - alias3: newMetricResultMap(0.3), - alias4: noSuchMetricMap(), - alias5: newMetricResultMap(1.1), - } - - t.Run("nonexistent", func(t *testing.T) { - worstMetric := aggregateTabletMetricResults(nonexistentMetricName, tabletResultsMap, 0, false, 0) - _, err := worstMetric.Get() - assert.Error(t, err) - assert.Equal(t, base.ErrNoSuchMetric, err) - }) - t.Run("no ignore", func(t *testing.T) { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) - _, err := worstMetric.Get() - assert.Error(t, err) - assert.Equal(t, base.ErrNoSuchMetric, err) - }) - t.Run("ignore 1", func(t *testing.T) { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, 1.7, value) - }) - t.Run("ignore 2", func(t *testing.T) { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, 1.2, value) - }) - - tabletResultsMap[alias1][base.DefaultMetricName] = base.NoSuchMetric - t.Run("no such metric", func(t *testing.T) { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 0, false, 0) - _, err := worstMetric.Get() - assert.Error(t, err) - assert.Equal(t, base.ErrNoSuchMetric, err) - }) - t.Run("no such metric, ignore 1", func(t *testing.T) { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 1, false, 0) - _, err := worstMetric.Get() - assert.Error(t, err) - assert.Equal(t, base.ErrNoSuchMetric, err) - }) - t.Run("metric found", func(t *testing.T) { - worstMetric := aggregateTabletMetricResults(base.DefaultMetricName, tabletResultsMap, 2, false, 0) - value, err := worstMetric.Get() - assert.NoError(t, err) - assert.Equal(t, value, 1.7) - }) -} diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 973c39dd842..f660358ed9f 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1254,7 +1254,7 @@ func (throttler *Throttler) aggregateMySQLMetrics(ctx context.Context) error { ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold ignoreDialTCPErrors := throttler.configSettings.MySQLStore.IgnoreDialTCPErrors - aggregatedMetric := aggregateTabletMetricResults(metricName, tabletResultsMap, ignoreHostsCount, ignoreDialTCPErrors, ignoreHostsThreshold) + aggregatedMetric := base.AggregateTabletMetricResults(metricName, tabletResultsMap, ignoreHostsCount, ignoreDialTCPErrors, ignoreHostsThreshold) aggregatedMetricName := metricName.AggregatedName(scope) throttler.aggregatedMetrics.Set(aggregatedMetricName, aggregatedMetric, cache.DefaultExpiration) if metricName == metricNameUsedAsDefault { From c82ea9a713b4c266c85345febfecec50e1ccf92a Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:47:38 +0300 Subject: [PATCH 14/36] remove ctx where unused Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/throttler.go | 4 ++-- go/vt/vttablet/tabletserver/throttle/throttler_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index f660358ed9f..e35fb288a14 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -980,7 +980,7 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { throttler.updateMySQLClusterProbes(ctx, probes) case <-mysqlAggregateTicker.C: if throttler.IsOpen() { - throttler.aggregateMySQLMetrics(ctx) + throttler.aggregateMySQLMetrics() } case <-throttledAppsTicker.C: if throttler.IsOpen() { @@ -1247,7 +1247,7 @@ func (throttler *Throttler) metricNameUsedAsDefault() base.MetricName { } // synchronous aggregation of collected data -func (throttler *Throttler) aggregateMySQLMetrics(ctx context.Context) error { +func (throttler *Throttler) aggregateMySQLMetrics() error { metricNameUsedAsDefault := throttler.metricNameUsedAsDefault() aggregateTabletsMetrics := func(scope base.Scope, metricName base.MetricName, tabletResultsMap base.TabletResultMap) { ignoreHostsCount := throttler.mysqlInventory.IgnoreHostsCount diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index c9025ca7e1f..ac4e65c304c 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -1142,7 +1142,7 @@ func TestProbesWhileOperating(t *testing.T) { // as opposed to choosing the "lag" metric results. throttler.customMetricsQuery.Store("select non_empty") <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.aggregateMySQLMetrics(ctx) + throttler.aggregateMySQLMetrics() }) assert.Equal(t, base.CustomMetricName, throttler.metricNameUsedAsDefault()) // throttler.aggregateMySQLMetrics(ctx) From 342378003a623ea26424e9e668a10d4887b16867 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:54:29 +0300 Subject: [PATCH 15/36] terminology: remove 'MySQL' where irrelevant. Add where relevant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/throttle/throttler.go | 80 +++++++++---------- .../tabletserver/throttle/throttler_test.go | 69 ++++++++-------- 2 files changed, 74 insertions(+), 75 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index e35fb288a14..266f9c31aa6 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -179,12 +179,12 @@ type Throttler struct { throttleTabletTypesMap map[topodatapb.TabletType]bool - throttleMetricChan chan *base.ThrottleMetric - mysqlClusterProbesChan chan *base.ClusterProbes - throttlerConfigChan chan *topodatapb.ThrottlerConfig - serialFuncChan chan func() // Used by unit tests to inject non-racy behavior + throttleMetricChan chan *base.ThrottleMetric + clusterProbesChan chan *base.ClusterProbes + throttlerConfigChan chan *topodatapb.ThrottlerConfig + serialFuncChan chan func() // Used by unit tests to inject non-racy behavior - mysqlInventory *base.Inventory + inventory *base.Inventory metricsQuery atomic.Value customMetricsQuery atomic.Value @@ -251,10 +251,10 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv } throttler.throttleMetricChan = make(chan *base.ThrottleMetric) - throttler.mysqlClusterProbesChan = make(chan *base.ClusterProbes) + throttler.clusterProbesChan = make(chan *base.ClusterProbes) throttler.throttlerConfigChan = make(chan *topodatapb.ThrottlerConfig) throttler.serialFuncChan = make(chan func()) - throttler.mysqlInventory = base.NewInventory() + throttler.inventory = base.NewInventory() throttler.throttledApps = cache.New(cache.NoExpiration, 0) throttler.metricThresholds = cache.New(cache.NoExpiration, 0) @@ -778,8 +778,8 @@ func (throttler *Throttler) readSelfLoadAvgPerCore(ctx context.Context) *base.Th return metric } -// readSelfThrottleMetric reads the mysql metric from thi very tablet's backend mysql. -func (throttler *Throttler) readSelfThrottleMetric(ctx context.Context, query string) *base.ThrottleMetric { +// readSelfMySQLThrottleMetric reads the metric from this very tablet or from its backend mysql. +func (throttler *Throttler) readSelfMySQLThrottleMetric(ctx context.Context, query string) *base.ThrottleMetric { metric := &base.ThrottleMetric{ Scope: base.SelfScope, Alias: throttler.tabletAlias, @@ -931,9 +931,9 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { if throttler.IsOpen() { // frequent // Always collect self metrics: - throttler.collectSelfMySQLMetrics(ctx, tmClient) + throttler.collectSelfMetrics(ctx) if !throttler.isDormant() { - throttler.collectShardMySQLMetrics(ctx, tmClient) + throttler.collectShardMetrics(ctx, tmClient) } // if throttler.recentlyChecked() { @@ -959,28 +959,28 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { if throttler.IsOpen() { // infrequent if throttler.isDormant() { - throttler.collectShardMySQLMetrics(ctx, tmClient) + throttler.collectShardMetrics(ctx, tmClient) } } case metric := <-throttler.throttleMetricChan: // incoming MySQL metric, frequent, as result of collectMySQLMetrics() - metricResultsMap, ok := throttler.mysqlInventory.TabletMetrics[metric.GetTabletAlias()] + metricResultsMap, ok := throttler.inventory.TabletMetrics[metric.GetTabletAlias()] if !ok { metricResultsMap = base.NewMetricResultMap() - throttler.mysqlInventory.TabletMetrics[metric.GetTabletAlias()] = metricResultsMap + throttler.inventory.TabletMetrics[metric.GetTabletAlias()] = metricResultsMap } metricResultsMap[metric.Name] = metric case <-mysqlRefreshTicker.C: // sparse if throttler.IsOpen() { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) } - case probes := <-throttler.mysqlClusterProbesChan: - // incoming structural update, sparse, as result of refreshMySQLInventory() - throttler.updateMySQLClusterProbes(ctx, probes) + case probes := <-throttler.clusterProbesChan: + // incoming structural update, sparse, as result of refreshInventory() + throttler.updateClusterProbes(ctx, probes) case <-mysqlAggregateTicker.C: if throttler.IsOpen() { - throttler.aggregateMySQLMetrics() + throttler.aggregateMetrics() } case <-throttledAppsTicker.C: if throttler.IsOpen() { @@ -1073,15 +1073,15 @@ func (throttler *Throttler) readSelfThrottleMetricsInternal(ctx context.Context) } } - go writeMetric(base.LagMetricName, throttler.readSelfThrottleMetric(ctx, sqlparser.BuildParsedQuery(defaultReplicationLagQuery, sidecar.GetIdentifier()).Query)) - go writeMetric(base.ThreadsRunningMetricName, throttler.readSelfThrottleMetric(ctx, threadsRunningQuery)) - go writeMetric(base.CustomMetricName, throttler.readSelfThrottleMetric(ctx, throttler.GetCustomMetricsQuery())) + go writeMetric(base.LagMetricName, throttler.readSelfMySQLThrottleMetric(ctx, sqlparser.BuildParsedQuery(defaultReplicationLagQuery, sidecar.GetIdentifier()).Query)) + go writeMetric(base.ThreadsRunningMetricName, throttler.readSelfMySQLThrottleMetric(ctx, threadsRunningQuery)) + go writeMetric(base.CustomMetricName, throttler.readSelfMySQLThrottleMetric(ctx, throttler.GetCustomMetricsQuery())) go writeMetric(base.LoadAvgMetricName, throttler.readSelfLoadAvgPerCore(ctx)) return nil } -func (throttler *Throttler) collectSelfMySQLMetrics(ctx context.Context, tmClient tmclient.TabletManagerClient) { - probe := throttler.mysqlInventory.ClustersProbes[throttler.tabletAlias] +func (throttler *Throttler) collectSelfMetrics(ctx context.Context) { + probe := throttler.inventory.ClustersProbes[throttler.tabletAlias] if probe == nil { // probe not created yet return @@ -1099,10 +1099,10 @@ func (throttler *Throttler) collectSelfMySQLMetrics(ctx context.Context, tmClien }() } -func (throttler *Throttler) collectShardMySQLMetrics(ctx context.Context, tmClient tmclient.TabletManagerClient) { +func (throttler *Throttler) collectShardMetrics(ctx context.Context, tmClient tmclient.TabletManagerClient) { // probes is known not to change. It can be *replaced*, but not changed. // so it's safe to iterate it - for _, probe := range throttler.mysqlInventory.ClustersProbes { + for _, probe := range throttler.inventory.ClustersProbes { if probe.Alias == throttler.tabletAlias { // We skip collecting our own metrics continue @@ -1130,8 +1130,8 @@ func (throttler *Throttler) collectShardMySQLMetrics(ctx context.Context, tmClie } } -// refreshMySQLInventory will re-structure the inventory based on reading config settings -func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { +// refreshInventory will re-structure the inventory based on reading config settings +func (throttler *Throttler) refreshInventory(ctx context.Context) error { // distribute the query/threshold from the throttler down to the cluster settings and from there to the probes addProbe := func(alias string, tablet *topodatapb.Tablet, scope base.Scope, mysqlSettings *config.MySQLConfigurationSettings, probes base.Probes) bool { for _, ignore := range mysqlSettings.IgnoreHosts { @@ -1164,7 +1164,7 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { select { case <-ctx.Done(): return ctx.Err() - case throttler.mysqlClusterProbesChan <- clusterProbes: + case throttler.clusterProbesChan <- clusterProbes: return nil } } @@ -1198,8 +1198,8 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { // of previous clusters it used to probe. It may have recollection of specific probes for such clusters. // This now ensures any existing cluster probes are overridden with an empty list of probes. // `clusterProbes` was created above as empty, and identifiable via `scope`. This will in turn - // be used to overwrite throttler.mysqlInventory.ClustersProbes[clusterProbes.scope] in - // updateMySQLClusterProbes(). + // be used to overwrite throttler.inventory.ClustersProbes[clusterProbes.scope] in + // updateClusterProbes(). return attemptWriteProbes(clusterProbes) // not the leader (primary tablet)? Then no more work for us. } @@ -1225,17 +1225,17 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { } go func() { if err := collect(); err != nil { - log.Errorf("refreshMySQLInventory: %+v", err) + log.Errorf("refreshInventory: %+v", err) } }() return nil } // synchronous update of inventory -func (throttler *Throttler) updateMySQLClusterProbes(ctx context.Context, clusterProbes *base.ClusterProbes) error { - throttler.mysqlInventory.ClustersProbes = clusterProbes.TabletProbes - throttler.mysqlInventory.IgnoreHostsCount = clusterProbes.IgnoreHostsCount - throttler.mysqlInventory.IgnoreHostsThreshold = clusterProbes.IgnoreHostsThreshold +func (throttler *Throttler) updateClusterProbes(ctx context.Context, clusterProbes *base.ClusterProbes) error { + throttler.inventory.ClustersProbes = clusterProbes.TabletProbes + throttler.inventory.IgnoreHostsCount = clusterProbes.IgnoreHostsCount + throttler.inventory.IgnoreHostsThreshold = clusterProbes.IgnoreHostsThreshold return nil } @@ -1247,11 +1247,11 @@ func (throttler *Throttler) metricNameUsedAsDefault() base.MetricName { } // synchronous aggregation of collected data -func (throttler *Throttler) aggregateMySQLMetrics() error { +func (throttler *Throttler) aggregateMetrics() error { metricNameUsedAsDefault := throttler.metricNameUsedAsDefault() aggregateTabletsMetrics := func(scope base.Scope, metricName base.MetricName, tabletResultsMap base.TabletResultMap) { - ignoreHostsCount := throttler.mysqlInventory.IgnoreHostsCount - ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold + ignoreHostsCount := throttler.inventory.IgnoreHostsCount + ignoreHostsThreshold := throttler.inventory.IgnoreHostsThreshold ignoreDialTCPErrors := throttler.configSettings.MySQLStore.IgnoreDialTCPErrors aggregatedMetric := base.AggregateTabletMetricResults(metricName, tabletResultsMap, ignoreHostsCount, ignoreDialTCPErrors, ignoreHostsThreshold) @@ -1268,7 +1268,7 @@ func (throttler *Throttler) aggregateMySQLMetrics() error { // is to be stored as "default" continue } - selfResultsMap, shardResultsMap := throttler.mysqlInventory.TabletMetrics.Split(throttler.tabletAlias) + selfResultsMap, shardResultsMap := throttler.inventory.TabletMetrics.Split(throttler.tabletAlias) aggregateTabletsMetrics(base.SelfScope, metricName, selfResultsMap) aggregateTabletsMetrics(base.ShardScope, metricName, shardResultsMap) } diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index ac4e65c304c..2e272d79e20 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -233,23 +233,23 @@ func newTestThrottler() *Throttler { env := tabletenv.NewEnv(vtenv.NewTestEnv(), nil, "TabletServerTest") throttler := &Throttler{ - mysqlClusterProbesChan: make(chan *base.ClusterProbes), - heartbeatWriter: &FakeHeartbeatWriter{}, - ts: &FakeTopoServer{}, - mysqlInventory: base.NewInventory(), - pool: connpool.NewPool(env, "ThrottlerPool", tabletenv.ConnPoolConfig{}), - tabletTypeFunc: func() topodatapb.TabletType { return topodatapb.TabletType_PRIMARY }, - overrideTmClient: &fakeTMClient{}, + clusterProbesChan: make(chan *base.ClusterProbes), + heartbeatWriter: &FakeHeartbeatWriter{}, + ts: &FakeTopoServer{}, + inventory: base.NewInventory(), + pool: connpool.NewPool(env, "ThrottlerPool", tabletenv.ConnPoolConfig{}), + tabletTypeFunc: func() topodatapb.TabletType { return topodatapb.TabletType_PRIMARY }, + overrideTmClient: &fakeTMClient{}, } throttler.metricsQuery.Store(metricsQuery) throttler.MetricsThreshold.Store(math.Float64bits(0.75)) throttler.configSettings = config.NewConfigurationSettings() throttler.initConfig() throttler.throttleMetricChan = make(chan *base.ThrottleMetric) - throttler.mysqlClusterProbesChan = make(chan *base.ClusterProbes) + throttler.clusterProbesChan = make(chan *base.ClusterProbes) throttler.throttlerConfigChan = make(chan *topodatapb.ThrottlerConfig) throttler.serialFuncChan = make(chan func()) - throttler.mysqlInventory = base.NewInventory() + throttler.inventory = base.NewInventory() throttler.throttledApps = cache.New(cache.NoExpiration, 0) throttler.metricThresholds = cache.New(cache.NoExpiration, 0) @@ -887,12 +887,12 @@ func TestIsAppExempted(t *testing.T) { assert.True(t, throttler.IsAppExempted("schema-tracker")) } -// TestRefreshMySQLInventory tests the behavior of the throttler's RefreshMySQLInventory() function, which +// TestRefreshInventory tests the behavior of the throttler's RefreshInventory() function, which // is called periodically in actual throttler. For a given cluster name, it generates a list of probes // the throttler will use to check metrics. // On a replica tablet, that list is expect to probe the tablet itself. // On the PRIMARY, the list includes all shard tablets, including the PRIMARY itself. -func TestRefreshMySQLInventory(t *testing.T) { +func TestRefreshInventory(t *testing.T) { ctx := context.Background() // for development, replace with ctx := utils.LeakCheckContext(t) ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -901,10 +901,10 @@ func TestRefreshMySQLInventory(t *testing.T) { configSettings := config.NewConfigurationSettings() throttler := &Throttler{ - mysqlClusterProbesChan: make(chan *base.ClusterProbes), - metricThresholds: cache.New(cache.NoExpiration, 0), - ts: &FakeTopoServer{}, - mysqlInventory: base.NewInventory(), + clusterProbesChan: make(chan *base.ClusterProbes), + metricThresholds: cache.New(cache.NoExpiration, 0), + ts: &FakeTopoServer{}, + inventory: base.NewInventory(), } throttler.metricsQuery.Store(metricsQuery) throttler.configSettings = configSettings @@ -927,12 +927,12 @@ func TestRefreshMySQLInventory(t *testing.T) { defer cancel() for { select { - case probes := <-throttler.mysqlClusterProbesChan: + case probes := <-throttler.clusterProbesChan: // Worth noting that in this unit test, the throttler is _closed_ and _disabled_. Its own Operate() function does - // not run, and therefore there is none but us to both populate `mysqlClusterProbesChan` as well as + // not run, and therefore there is none but us to both populate `clusterProbesChan` as well as // read from it. We do not compete here with any other goroutine. assert.NotNil(t, probes) - throttler.updateMySQLClusterProbes(ctx, probes) + throttler.updateClusterProbes(ctx, probes) validateProbesCount(t, probes.TabletProbes) // Achieved our goal return @@ -942,7 +942,7 @@ func TestRefreshMySQLInventory(t *testing.T) { } }) t.Run("validating probes", func(t *testing.T) { - probes := throttler.mysqlInventory.ClustersProbes + probes := throttler.inventory.ClustersProbes validateProbesCount(t, probes) }) }) @@ -950,19 +950,19 @@ func TestRefreshMySQLInventory(t *testing.T) { t.Run("initial, not leader", func(t *testing.T) { throttler.isLeader.Store(false) - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) validateClusterProbes(t, ctx) }) t.Run("promote", func(t *testing.T) { throttler.isLeader.Store(true) - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) validateClusterProbes(t, ctx) }) t.Run("demote, expect cleanup", func(t *testing.T) { throttler.isLeader.Store(false) - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) validateClusterProbes(t, ctx) }) } @@ -1103,7 +1103,7 @@ func TestProbesWhileOperating(t *testing.T) { client := NewBackgroundClient(throttler, testAppName, base.UndefinedScope) t.Run("threshold exceeded", func(t *testing.T) { <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) }) { checkOK := client.ThrottleCheckOK(ctx, "") @@ -1115,7 +1115,7 @@ func TestProbesWhileOperating(t *testing.T) { t.Run("adjust threshold", func(t *testing.T) { throttler.MetricsThreshold.Store(math.Float64bits(0.95)) <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) }) { checkOK := client.ThrottleCheckOK(ctx, "") @@ -1125,7 +1125,7 @@ func TestProbesWhileOperating(t *testing.T) { t.Run("restore threshold", func(t *testing.T) { throttler.MetricsThreshold.Store(savedThreshold) <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) }) client.clearSuccessfulResultsCache() // ensure we don't read the successful result from the test above { @@ -1142,10 +1142,9 @@ func TestProbesWhileOperating(t *testing.T) { // as opposed to choosing the "lag" metric results. throttler.customMetricsQuery.Store("select non_empty") <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.aggregateMySQLMetrics() + throttler.aggregateMetrics() }) assert.Equal(t, base.CustomMetricName, throttler.metricNameUsedAsDefault()) - // throttler.aggregateMySQLMetrics(ctx) aggr := throttler.aggregatedMetricsSnapshot() assert.Equalf(t, 2*len(base.KnownMetricNames), len(aggr), "aggregated: %+v", aggr) // "self" and "shard", per known metric assert.Equal(t, 2*len(base.KnownMetricNames), throttler.aggregatedMetrics.ItemCount()) // flushed upon Disable() @@ -1193,7 +1192,7 @@ func TestProbesWhileOperating(t *testing.T) { client := NewBackgroundClient(throttler, testAppName, base.UndefinedScope) t.Run("threshold exceeded", func(t *testing.T) { <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) }) { checkOK := client.ThrottleCheckOK(ctx, "") @@ -1205,7 +1204,7 @@ func TestProbesWhileOperating(t *testing.T) { t.Run("adjust threshold, too low", func(t *testing.T) { throttler.MetricsThreshold.Store(math.Float64bits(0.95)) <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) }) { checkOK := client.ThrottleCheckOK(ctx, "") @@ -1215,7 +1214,7 @@ func TestProbesWhileOperating(t *testing.T) { t.Run("adjust threshold, still too low", func(t *testing.T) { throttler.MetricsThreshold.Store(math.Float64bits(15)) <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) }) { checkOK := client.ThrottleCheckOK(ctx, "") @@ -1225,7 +1224,7 @@ func TestProbesWhileOperating(t *testing.T) { t.Run("adjust threshold", func(t *testing.T) { throttler.MetricsThreshold.Store(math.Float64bits(18)) <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) }) { checkOK := client.ThrottleCheckOK(ctx, "") @@ -1235,7 +1234,7 @@ func TestProbesWhileOperating(t *testing.T) { t.Run("restore threshold", func(t *testing.T) { throttler.MetricsThreshold.Store(savedThreshold) <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) }) client.clearSuccessfulResultsCache() // ensure we don't read the successful result from the test above { @@ -1340,7 +1339,7 @@ func TestProbesPostDisable(t *testing.T) { throttler := newTestThrottler() runThrottler(t, ctx, throttler, 2*time.Second, nil) - probes := throttler.mysqlInventory.ClustersProbes + probes := throttler.inventory.ClustersProbes <-time.After(1 * time.Second) // throttler's context was cancelled, but still some functionality needs to complete t.Run("probes", func(t *testing.T) { @@ -1360,7 +1359,7 @@ func TestProbesPostDisable(t *testing.T) { }) t.Run("metrics", func(t *testing.T) { - assert.Equal(t, 3, len(throttler.mysqlInventory.TabletMetrics)) // 1 self tablet + 2 shard tablets + assert.Equal(t, 3, len(throttler.inventory.TabletMetrics)) // 1 self tablet + 2 shard tablets }) t.Run("aggregated", func(t *testing.T) { @@ -1868,7 +1867,7 @@ func TestReplica(t *testing.T) { // Change custom threshold throttler.MetricsThreshold.Store(math.Float64bits(0.1)) <-runSerialFunction(t, ctx, throttler, func(ctx context.Context) { - throttler.refreshMySQLInventory(ctx) + throttler.refreshInventory(ctx) }) checkResult = throttler.Check(ctx, testAppName.String(), base.KnownMetricNames, flags) require.NotNil(t, checkResult) From 1f584a1faf45a6bf6208c0239bd9a27dad063878 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:55:54 +0300 Subject: [PATCH 16/36] terminology: remove 'MySQL' where irrelevant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/throttle/throttler.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 266f9c31aa6..564651fff94 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -861,10 +861,10 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { return t } leaderCheckTicker := addTicker(throttler.leaderCheckInterval) - mysqlCollectTicker := addTicker(throttler.activeCollectInterval) - mysqlDormantCollectTicker := addTicker(throttler.dormantCollectInterval) - mysqlRefreshTicker := addTicker(throttler.inventoryRefreshInterval) - mysqlAggregateTicker := addTicker(throttler.metricsAggregateInterval) + activeCollectTicker := addTicker(throttler.activeCollectInterval) + dormantCollectTicker := addTicker(throttler.dormantCollectInterval) + inventoryRefreshTicker := addTicker(throttler.inventoryRefreshInterval) + metricsAggregateTicker := addTicker(throttler.metricsAggregateInterval) throttledAppsTicker := addTicker(throttler.throttledAppsSnapshotInterval) primaryStimulatorRateLimiter := timer.NewRateLimiter(throttler.dormantPeriod) throttler.recentCheckRateLimiter = timer.NewRateLimiter(recentCheckRateLimiterInterval) @@ -923,11 +923,11 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { if transitionedIntoLeader { // transitioned into leadership, let's speed up the next 'refresh' and 'collect' ticks - go mysqlRefreshTicker.TickNow() + go inventoryRefreshTicker.TickNow() throttler.requestHeartbeats() } }() - case <-mysqlCollectTicker.C: + case <-activeCollectTicker.C: if throttler.IsOpen() { // frequent // Always collect self metrics: @@ -955,7 +955,7 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { } } - case <-mysqlDormantCollectTicker.C: + case <-dormantCollectTicker.C: if throttler.IsOpen() { // infrequent if throttler.isDormant() { @@ -963,14 +963,14 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { } } case metric := <-throttler.throttleMetricChan: - // incoming MySQL metric, frequent, as result of collectMySQLMetrics() + // incoming metric, frequent, as result of collectMetrics() metricResultsMap, ok := throttler.inventory.TabletMetrics[metric.GetTabletAlias()] if !ok { metricResultsMap = base.NewMetricResultMap() throttler.inventory.TabletMetrics[metric.GetTabletAlias()] = metricResultsMap } metricResultsMap[metric.Name] = metric - case <-mysqlRefreshTicker.C: + case <-inventoryRefreshTicker.C: // sparse if throttler.IsOpen() { throttler.refreshInventory(ctx) @@ -978,7 +978,7 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) { case probes := <-throttler.clusterProbesChan: // incoming structural update, sparse, as result of refreshInventory() throttler.updateClusterProbes(ctx, probes) - case <-mysqlAggregateTicker.C: + case <-metricsAggregateTicker.C: if throttler.IsOpen() { throttler.aggregateMetrics() } From 85994ed79de19caca6ee3f5c713db38a95f344c5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:57:02 +0300 Subject: [PATCH 17/36] renamed getMySQLStoreMetric -> getScopedMetric Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/check.go | 2 +- go/vt/vttablet/tabletserver/throttle/throttler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/check.go b/go/vt/vttablet/tabletserver/throttle/check.go index 069a1c275f5..e43c4cab043 100644 --- a/go/vt/vttablet/tabletserver/throttle/check.go +++ b/go/vt/vttablet/tabletserver/throttle/check.go @@ -163,7 +163,7 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, scope ba } metricResultFunc := func() (metricResult base.MetricResult, threshold float64) { - return check.throttler.getMySQLStoreMetric(ctx, metricScope, metricName) + return check.throttler.getScopedMetric(metricScope, metricName) } metricCheckResult := check.checkAppMetricResult(ctx, appName, metricResultFunc, flags) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 564651fff94..49a817c5952 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1282,7 +1282,7 @@ func (throttler *Throttler) getAggregatedMetric(aggregatedName string) base.Metr return base.NoSuchMetric } -func (throttler *Throttler) getMySQLStoreMetric(ctx context.Context, scope base.Scope, metricName base.MetricName) (base.MetricResult, float64) { +func (throttler *Throttler) getScopedMetric(scope base.Scope, metricName base.MetricName) (base.MetricResult, float64) { thresholdVal, found := throttler.metricThresholds.Get(metricName.String()) if !found { return base.NoSuchMetric, 0 From 2492d6d9504b50af35ef2861179c33faa7cbf848 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:53:20 +0300 Subject: [PATCH 18/36] comment wording Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/throttler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 49a817c5952..73458974dcb 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1527,7 +1527,7 @@ func (throttler *Throttler) AppRequestMetricResult(ctx context.Context, appName return metricResultFunc() } -// checkScope checks the aggregated value of given MySQL store +// checkScope checks the aggregated value of given store func (throttler *Throttler) checkScope(ctx context.Context, appName string, scope base.Scope, metricNames base.MetricNames, flags *CheckFlags) (checkResult *CheckResult) { if !throttler.IsRunning() { return okMetricCheckResult From d7c63c8af3edaf10f48e6af4ae46458718678ae1 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 14:04:30 +0300 Subject: [PATCH 19/36] tests: explicitly disabling throttler so as to ensure it does not re-enable Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/throttler_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index 2e272d79e20..6363143fd43 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -770,6 +770,16 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.Len(t, checkResult.Metrics, 1) }) }) + + t.Run("Disable", func(t *testing.T) { + throttlerConfig := &topodatapb.ThrottlerConfig{ + Enabled: false, + MetricThresholds: map[string]float64{}, + AppCheckedMetrics: map[string]*topodatapb.ThrottlerConfig_MetricNames{}, + } + throttler.applyThrottlerConfig(ctx, throttlerConfig) + sleepTillThresholdApplies() + }) }) } From 114405f2ed774fbc221c43098c09e34e1df24a90 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:35:23 +0300 Subject: [PATCH 20/36] CheckThrottlerResponse: include the matched app name Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletmanagerdata/tabletmanagerdata.pb.go | 317 +++++++++--------- .../tabletmanagerdata_vtproto.pb.go | 44 +++ proto/tabletmanagerdata.proto | 3 + web/vtadmin/src/proto/vtadmin.d.ts | 6 + web/vtadmin/src/proto/vtadmin.js | 23 ++ 5 files changed, 240 insertions(+), 153 deletions(-) diff --git a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go index c0896882735..133475bde34 100644 --- a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go +++ b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go @@ -6636,6 +6636,8 @@ type CheckThrottlerResponse struct { // Metrics is a map (metric name -> metric value/error) so that the client has as much // information as possible about all the checked metrics. Metrics map[string]*CheckThrottlerResponse_Metric `protobuf:"bytes,7,rep,name=metrics,proto3" json:"metrics,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // AppName is the name of app that was matched by the throttler + AppName string `protobuf:"bytes,8,opt,name=app_name,json=appName,proto3" json:"app_name,omitempty"` } func (x *CheckThrottlerResponse) Reset() { @@ -6719,6 +6721,13 @@ func (x *CheckThrottlerResponse) GetMetrics() map[string]*CheckThrottlerResponse return nil } +func (x *CheckThrottlerResponse) GetAppName() string { + if x != nil { + return x.AppName + } + return "" +} + type GetThrottlerStatusRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -8211,7 +8220,7 @@ var file_tabletmanagerdata_proto_rawDesc = []byte{ 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0xc2, 0x04, 0x0a, 0x16, + 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0xdd, 0x04, 0x0a, 0x16, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x74, 0x61, @@ -8229,161 +8238,163 @@ var file_tabletmanagerdata_proto_rawDesc = []byte{ 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0xb7, 0x01, 0x0a, 0x06, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1c, - 0x0a, 0x09, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x01, 0x52, 0x09, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x14, 0x0a, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x63, 0x6f, - 0x70, 0x65, 0x1a, 0x6c, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x46, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x54, 0x68, 0x72, - 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x1b, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xe1, 0x0f, - 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, - 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x17, - 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x6f, 0x70, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x06, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x65, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x64, 0x6f, 0x72, - 0x6d, 0x61, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x44, 0x6f, - 0x72, 0x6d, 0x61, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x6c, 0x61, 0x67, 0x5f, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0e, 0x6c, 0x61, 0x67, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, - 0x2e, 0x0a, 0x13, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x63, 0x75, - 0x73, 0x74, 0x6f, 0x6d, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, - 0x2b, 0x0a, 0x11, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, - 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x64, 0x65, 0x66, 0x61, - 0x75, 0x6c, 0x74, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x3c, 0x0a, 0x1b, - 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65, 0x64, - 0x5f, 0x61, 0x73, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x17, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x55, 0x73, 0x65, - 0x64, 0x41, 0x73, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x73, 0x0a, 0x12, 0x61, 0x67, - 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, - 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x61, 0x67, - 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, - 0x70, 0x0a, 0x11, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, - 0x6f, 0x6c, 0x64, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, + 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x70, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x70, 0x4e, + 0x61, 0x6d, 0x65, 0x1a, 0xb7, 0x01, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, + 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x72, + 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x74, 0x68, + 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, + 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x1a, 0x6c, 0x0a, + 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x46, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, + 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1b, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x10, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, - 0x73, 0x12, 0x67, 0x0a, 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x68, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, - 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x67, 0x0a, 0x0e, 0x74, 0x68, - 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x5f, 0x61, 0x70, 0x70, 0x73, 0x18, 0x0f, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, - 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, - 0x70, 0x70, 0x73, 0x12, 0x74, 0x0a, 0x13, 0x61, 0x70, 0x70, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, - 0x65, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x44, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, - 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x41, 0x70, 0x70, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x61, 0x70, 0x70, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x63, - 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x18, 0x11, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0f, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x65, 0x64, 0x12, 0x5e, 0x0a, 0x0b, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x61, - 0x70, 0x70, 0x73, 0x18, 0x12, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, - 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, - 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, - 0x41, 0x70, 0x70, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x1a, 0x80, 0x01, 0x0a, 0x16, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x4d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x50, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x1a, 0x43, 0x0a, 0x15, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x68, 0x72, - 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x81, 0x01, 0x0a, 0x0c, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x34, 0x0a, 0x0f, 0x6c, 0x61, 0x73, - 0x74, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x52, 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x41, 0x74, 0x12, - 0x3b, 0x0a, 0x1a, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, - 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x17, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x53, 0x69, 0x6e, 0x63, - 0x65, 0x4c, 0x61, 0x73, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x1a, 0x7c, 0x0a, 0x12, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x50, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, - 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5c, 0x0a, 0x12, 0x54, 0x68, - 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x68, 0x72, - 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x44, 0x0a, 0x16, 0x41, 0x70, 0x70, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x59, - 0x0a, 0x09, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x12, 0x2b, 0x0a, 0x0a, 0x63, - 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0c, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x09, 0x63, - 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x1a, 0x76, 0x0a, 0x0f, 0x52, 0x65, 0x63, - 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x4d, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xe1, 0x0f, 0x0a, 0x1a, 0x47, 0x65, 0x74, + 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1b, 0x0a, 0x09, + 0x69, 0x73, 0x5f, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, + 0x6f, 0x70, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4f, 0x70, + 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x64, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x74, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x44, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x74, + 0x12, 0x28, 0x0a, 0x10, 0x6c, 0x61, 0x67, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x61, 0x67, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x2e, 0x0a, 0x13, 0x63, 0x75, + 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x64, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, + 0x0a, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x54, 0x68, + 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x3c, 0x0a, 0x1b, 0x6d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x61, 0x73, 0x5f, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x17, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x55, 0x73, 0x65, 0x64, 0x41, 0x73, 0x44, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x73, 0x0a, 0x12, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, + 0x74, 0x65, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x44, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, + 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, + 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x70, 0x0a, 0x11, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x18, + 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, + 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x6d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x12, 0x67, 0x0a, 0x0e, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x18, 0x0e, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, + 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x67, 0x0a, 0x0e, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, + 0x65, 0x64, 0x5f, 0x61, 0x70, 0x70, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x63, - 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x2a, 0x3e, 0x0a, 0x19, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x53, 0x65, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x07, - 0x0a, 0x03, 0x41, 0x4e, 0x59, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x4f, 0x52, 0x44, - 0x45, 0x52, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, - 0x03, 0x42, 0x30, 0x5a, 0x2e, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, - 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, - 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x68, 0x72, + 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x0d, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x12, 0x74, + 0x0a, 0x13, 0x61, 0x70, 0x70, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x11, 0x61, 0x70, 0x70, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x6c, 0x79, + 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, + 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x12, + 0x5e, 0x0a, 0x0b, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x70, 0x70, 0x73, 0x18, 0x12, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, + 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x0a, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x73, 0x1a, + 0x3a, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x1a, 0x80, 0x01, 0x0a, 0x16, + 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x50, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, + 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x43, + 0x0a, 0x15, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, + 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x81, 0x01, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x48, 0x65, + 0x61, 0x6c, 0x74, 0x68, 0x12, 0x34, 0x0a, 0x0f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, + 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x0d, 0x6c, 0x61, 0x73, + 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x41, 0x74, 0x12, 0x3b, 0x0a, 0x1a, 0x73, 0x65, + 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6c, 0x61, 0x73, 0x74, + 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x17, + 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x4c, 0x61, 0x73, 0x74, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x1a, 0x7c, 0x0a, 0x12, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x50, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, + 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5c, 0x0a, 0x12, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, + 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x74, + 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, + 0x64, 0x41, 0x70, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x44, 0x0a, 0x16, 0x41, 0x70, 0x70, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, + 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x59, 0x0a, 0x09, 0x52, 0x65, 0x63, + 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x12, 0x2b, 0x0a, 0x0a, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, + 0x64, 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x74, 0x74, + 0x69, 0x6d, 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x09, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, + 0x64, 0x41, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, + 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x43, 0x6f, 0x64, 0x65, 0x1a, 0x76, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, + 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x4d, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, + 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, + 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, 0x3e, 0x0a, 0x19, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, + 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4e, 0x59, + 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x4f, 0x52, 0x44, 0x45, 0x52, 0x10, 0x01, 0x12, + 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x42, 0x30, 0x5a, 0x2e, + 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, + 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/go/vt/proto/tabletmanagerdata/tabletmanagerdata_vtproto.pb.go b/go/vt/proto/tabletmanagerdata/tabletmanagerdata_vtproto.pb.go index d48882613d1..143519b063c 100644 --- a/go/vt/proto/tabletmanagerdata/tabletmanagerdata_vtproto.pb.go +++ b/go/vt/proto/tabletmanagerdata/tabletmanagerdata_vtproto.pb.go @@ -2529,6 +2529,7 @@ func (m *CheckThrottlerResponse) CloneVT() *CheckThrottlerResponse { Error: m.Error, Message: m.Message, RecentlyChecked: m.RecentlyChecked, + AppName: m.AppName, } if rhs := m.Metrics; rhs != nil { tmpContainer := make(map[string]*CheckThrottlerResponse_Metric, len(rhs)) @@ -8833,6 +8834,13 @@ func (m *CheckThrottlerResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.AppName) > 0 { + i -= len(m.AppName) + copy(dAtA[i:], m.AppName) + i = encodeVarint(dAtA, i, uint64(len(m.AppName))) + i-- + dAtA[i] = 0x42 + } if len(m.Metrics) > 0 { for k := range m.Metrics { v := m.Metrics[k] @@ -11566,6 +11574,10 @@ func (m *CheckThrottlerResponse) SizeVT() (n int) { n += mapEntrySize + 1 + sov(uint64(mapEntrySize)) } } + l = len(m.AppName) + if l > 0 { + n += 1 + l + sov(uint64(l)) + } n += len(m.unknownFields) return n } @@ -25594,6 +25606,38 @@ func (m *CheckThrottlerResponse) UnmarshalVT(dAtA []byte) error { } m.Metrics[mapkey] = mapvalue iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AppName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AppName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skip(dAtA[iNdEx:]) diff --git a/proto/tabletmanagerdata.proto b/proto/tabletmanagerdata.proto index ffe4aa29abf..76ce4b0b9ad 100644 --- a/proto/tabletmanagerdata.proto +++ b/proto/tabletmanagerdata.proto @@ -771,6 +771,9 @@ message CheckThrottlerResponse { // Metrics is a map (metric name -> metric value/error) so that the client has as much // information as possible about all the checked metrics. map metrics = 7; + + // AppName is the name of app that was matched by the throttler + string app_name = 8; } message GetThrottlerStatusRequest { diff --git a/web/vtadmin/src/proto/vtadmin.d.ts b/web/vtadmin/src/proto/vtadmin.d.ts index 7cec9343085..b7bb5fa3167 100644 --- a/web/vtadmin/src/proto/vtadmin.d.ts +++ b/web/vtadmin/src/proto/vtadmin.d.ts @@ -30653,6 +30653,9 @@ export namespace tabletmanagerdata { /** CheckThrottlerResponse metrics */ metrics?: ({ [k: string]: tabletmanagerdata.CheckThrottlerResponse.IMetric }|null); + + /** CheckThrottlerResponse app_name */ + app_name?: (string|null); } /** Represents a CheckThrottlerResponse. */ @@ -30685,6 +30688,9 @@ export namespace tabletmanagerdata { /** CheckThrottlerResponse metrics. */ public metrics: { [k: string]: tabletmanagerdata.CheckThrottlerResponse.IMetric }; + /** CheckThrottlerResponse app_name. */ + public app_name: string; + /** * Creates a new CheckThrottlerResponse instance using the specified properties. * @param [properties] Properties to set diff --git a/web/vtadmin/src/proto/vtadmin.js b/web/vtadmin/src/proto/vtadmin.js index 9fd83e98430..64bb2c71766 100644 --- a/web/vtadmin/src/proto/vtadmin.js +++ b/web/vtadmin/src/proto/vtadmin.js @@ -71283,6 +71283,7 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { * @property {string|null} [message] CheckThrottlerResponse message * @property {boolean|null} [recently_checked] CheckThrottlerResponse recently_checked * @property {Object.|null} [metrics] CheckThrottlerResponse metrics + * @property {string|null} [app_name] CheckThrottlerResponse app_name */ /** @@ -71357,6 +71358,14 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { */ CheckThrottlerResponse.prototype.metrics = $util.emptyObject; + /** + * CheckThrottlerResponse app_name. + * @member {string} app_name + * @memberof tabletmanagerdata.CheckThrottlerResponse + * @instance + */ + CheckThrottlerResponse.prototype.app_name = ""; + /** * Creates a new CheckThrottlerResponse instance using the specified properties. * @function create @@ -71398,6 +71407,8 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { writer.uint32(/* id 7, wireType 2 =*/58).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]); $root.tabletmanagerdata.CheckThrottlerResponse.Metric.encode(message.metrics[keys[i]], writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim().ldelim(); } + if (message.app_name != null && Object.hasOwnProperty.call(message, "app_name")) + writer.uint32(/* id 8, wireType 2 =*/66).string(message.app_name); return writer; }; @@ -71479,6 +71490,10 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { message.metrics[key] = value; break; } + case 8: { + message.app_name = reader.string(); + break; + } default: reader.skipType(tag & 7); break; @@ -71542,6 +71557,9 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { return "metrics." + error; } } + if (message.app_name != null && message.hasOwnProperty("app_name")) + if (!$util.isString(message.app_name)) + return "app_name: string expected"; return null; }; @@ -71579,6 +71597,8 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { message.metrics[keys[i]] = $root.tabletmanagerdata.CheckThrottlerResponse.Metric.fromObject(object.metrics[keys[i]]); } } + if (object.app_name != null) + message.app_name = String(object.app_name); return message; }; @@ -71604,6 +71624,7 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { object.error = ""; object.message = ""; object.recently_checked = false; + object.app_name = ""; } if (message.status_code != null && message.hasOwnProperty("status_code")) object.status_code = message.status_code; @@ -71623,6 +71644,8 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { for (let j = 0; j < keys2.length; ++j) object.metrics[keys2[j]] = $root.tabletmanagerdata.CheckThrottlerResponse.Metric.toObject(message.metrics[keys2[j]], options); } + if (message.app_name != null && message.hasOwnProperty("app_name")) + object.app_name = message.app_name; return object; }; From 692f7d52136ab32a3b92366df435e755f3a5b6e1 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:36:05 +0300 Subject: [PATCH 21/36] CheckThrottler: return matched app name Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/check.go | 8 +- .../tabletserver/throttle/check_result.go | 11 +- .../tabletserver/throttle/throttler.go | 50 +-- .../tabletserver/throttle/throttler_test.go | 291 ++++++++++++++---- 4 files changed, 277 insertions(+), 83 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/check.go b/go/vt/vttablet/tabletserver/throttle/check.go index e43c4cab043..8d61fddefdc 100644 --- a/go/vt/vttablet/tabletserver/throttle/check.go +++ b/go/vt/vttablet/tabletserver/throttle/check.go @@ -94,13 +94,13 @@ func (check *ThrottlerCheck) checkAppMetricResult(ctx context.Context, appName s // Handle deprioritized app logic denyApp := false // - metricResult, threshold := check.throttler.AppRequestMetricResult(ctx, appName, metricResultFunc, denyApp) + metricResult, threshold, matchedApp := check.throttler.AppRequestMetricResult(ctx, appName, metricResultFunc, denyApp) if flags.OverrideThreshold > 0 { threshold = flags.OverrideThreshold } value, err := metricResult.Get() if appName == "" { - return NewCheckResult(http.StatusExpectationFailed, value, threshold, fmt.Errorf("no app indicated")) + return NewCheckResult(http.StatusExpectationFailed, value, threshold, "", fmt.Errorf("no app indicated")) } var statusCode int @@ -123,7 +123,7 @@ func (check *ThrottlerCheck) checkAppMetricResult(ctx context.Context, appName s // all good! statusCode = http.StatusOK // 200 } - return NewCheckResult(statusCode, value, threshold, err) + return NewCheckResult(statusCode, value, threshold, matchedApp, err) } // Check is the core function that runs when a user wants to check a metric @@ -142,6 +142,7 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, scope ba checkResult.Threshold = metric.Threshold checkResult.Error = metric.Error checkResult.Message = metric.Message + checkResult.AppName = metric.AppName } for _, metricName := range metricNames { // Make sure not to modify the given scope. We create a new scope variable to work with. @@ -190,6 +191,7 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, scope ba Threshold: metricCheckResult.Threshold, Error: metricCheckResult.Error, Message: metricCheckResult.Message, + AppName: metricCheckResult.AppName, Scope: metricScope.String(), // This reports back the actual scope used for the check } checkResult.Metrics[metricName.String()] = metric diff --git a/go/vt/vttablet/tabletserver/throttle/check_result.go b/go/vt/vttablet/tabletserver/throttle/check_result.go index 3c8852e4042..2025f205145 100644 --- a/go/vt/vttablet/tabletserver/throttle/check_result.go +++ b/go/vt/vttablet/tabletserver/throttle/check_result.go @@ -54,6 +54,7 @@ type MetricResult struct { Threshold float64 `json:"Threshold"` Error error `json:"-"` Message string `json:"Message"` + AppName string `json:"AppName"` } // CheckResult is the result for an app inquiring on a metric. It also exports as JSON via the API @@ -64,15 +65,17 @@ type CheckResult struct { Error error `json:"-"` Message string `json:"Message"` RecentlyChecked bool `json:"RecentlyChecked"` + AppName string `json:"AppName"` Metrics map[string]*MetricResult `json:"Metrics"` // New in multi-metrics support. Will eventually replace the above fields. } // NewCheckResult returns a CheckResult -func NewCheckResult(statusCode int, value float64, threshold float64, err error) *CheckResult { +func NewCheckResult(statusCode int, value float64, threshold float64, appName string, err error) *CheckResult { result := &CheckResult{ StatusCode: statusCode, Value: value, Threshold: threshold, + AppName: appName, Error: err, } if err != nil { @@ -87,12 +90,10 @@ func (c *CheckResult) IsOK() bool { // NewErrorCheckResult returns a check result that indicates an error func NewErrorCheckResult(statusCode int, err error) *CheckResult { - return NewCheckResult(statusCode, 0, 0, err) + return NewCheckResult(statusCode, 0, 0, "", err) } // NoSuchMetricCheckResult is a result returns when a metric is unknown var NoSuchMetricCheckResult = NewErrorCheckResult(http.StatusNotFound, base.ErrNoSuchMetric) -var okMetricCheckResult = NewCheckResult(http.StatusOK, 0, 0, nil) - -var invalidCheckTypeCheckResult = NewErrorCheckResult(http.StatusInternalServerError, base.ErrInvalidCheckType) +var okMetricCheckResult = NewCheckResult(http.StatusOK, 0, 0, "", nil) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 73458974dcb..3ef367d60b2 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1378,8 +1378,8 @@ func (throttler *Throttler) UnthrottleApp(appName string) (appThrottle *base.App // IsAppThrottled tells whether some app should be throttled. // Assuming an app is throttled to some extend, it will randomize the result based // on the throttle ratio -func (throttler *Throttler) IsAppThrottled(appName string) bool { - appFound := false +func (throttler *Throttler) IsAppThrottled(appName string) (bool, string) { + appFound := "" isSingleAppNameThrottled := func(singleAppName string) bool { object, found := throttler.throttledApps.Get(singleAppName) if !found { @@ -1392,7 +1392,7 @@ func (throttler *Throttler) IsAppThrottled(appName string) bool { } // From this point on, we consider that this app has some throttling configuration // of any sort. - appFound = true + appFound = singleAppName if appThrottle.Exempt { return false } @@ -1403,32 +1403,32 @@ func (throttler *Throttler) IsAppThrottled(appName string) bool { return false } if isSingleAppNameThrottled(appName) { - return true + return true, appName } for _, singleAppName := range throttlerapp.Name(appName).SplitStrings() { if singleAppName == "" { continue } if isSingleAppNameThrottled(singleAppName) { - return true + return true, singleAppName } } // If app was found then there was some explicit throttle instruction for the app, and the app // passed the test. - if appFound { - return false + if appFound != "" { + return false, appFound } // If the app was not found, ie no specific throttle instruction was found for the app, then // the app should also consider the case where the "all" app is throttled. if isSingleAppNameThrottled(throttlerapp.AllName.String()) { // Means the "all" app is throttled. This is a special case, and it means "all apps are throttled" - return true + return true, throttlerapp.AllName.String() } - return false + return false, appName } // IsAppExempt -func (throttler *Throttler) IsAppExempted(appName string) bool { +func (throttler *Throttler) IsAppExempted(appName string) (bool, string) { isSingleAppNameExempted := func(singleAppName string) bool { if throttlerapp.ExemptFromChecks(appName) { // well known statically exempted apps return true @@ -1448,22 +1448,24 @@ func (throttler *Throttler) IsAppExempted(appName string) bool { return false } if isSingleAppNameExempted(appName) { - return true + return true, appName } for _, singleAppName := range throttlerapp.Name(appName).SplitStrings() { if singleAppName == "" { continue } if isSingleAppNameExempted(singleAppName) { - return true + return true, singleAppName } } - if isSingleAppNameExempted(throttlerapp.AllName.String()) && !throttler.IsAppThrottled(appName) { - return true + if isSingleAppNameExempted(throttlerapp.AllName.String()) { + if throttled, _ := throttler.IsAppThrottled(appName); !throttled { + return true, throttlerapp.AllName.String() + } } - return false + return false, appName } // ThrottledAppsMap returns a (copy) map of currently throttled apps @@ -1517,14 +1519,16 @@ func (throttler *Throttler) metricsHealthSnapshot() base.MetricHealthMap { } // AppRequestMetricResult gets a metric result in the context of a specific app -func (throttler *Throttler) AppRequestMetricResult(ctx context.Context, appName string, metricResultFunc base.MetricResultFunc, denyApp bool) (metricResult base.MetricResult, threshold float64) { +func (throttler *Throttler) AppRequestMetricResult(ctx context.Context, appName string, metricResultFunc base.MetricResultFunc, denyApp bool) (metricResult base.MetricResult, threshold float64, matchedApp string) { if denyApp { - return base.AppDeniedMetric, 0 + return base.AppDeniedMetric, 0, appName } - if throttler.IsAppThrottled(appName) { - return base.AppDeniedMetric, 0 + throttled, matchedApp := throttler.IsAppThrottled(appName) + if throttled { + return base.AppDeniedMetric, 0, matchedApp } - return metricResultFunc() + metricResult, threshold = metricResultFunc() + return metricResult, threshold, matchedApp } // checkScope checks the aggregated value of given store @@ -1532,10 +1536,12 @@ func (throttler *Throttler) checkScope(ctx context.Context, appName string, scop if !throttler.IsRunning() { return okMetricCheckResult } - if throttler.IsAppExempted(appName) { + if exempted, matchedApp := throttler.IsAppExempted(appName); exempted { // Some apps are exempt from checks. They are always responded with OK. This is because those apps are // continuous and do not generate a substantial load. - return okMetricCheckResult + result := okMetricCheckResult + result.AppName = matchedApp + return result } if len(metricNames) == 0 { diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index 6363143fd43..accea016b1e 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -790,10 +790,26 @@ func TestIsAppThrottled(t *testing.T) { heartbeatWriter: &FakeHeartbeatWriter{}, } t.Run("initial", func(t *testing.T) { - assert.False(t, throttler.IsAppThrottled("app1")) - assert.False(t, throttler.IsAppThrottled("app2")) - assert.False(t, throttler.IsAppThrottled("app3")) - assert.False(t, throttler.IsAppThrottled("app4")) + { + throttled, app := throttler.IsAppThrottled("app1") + assert.False(t, throttled) + assert.Equal(t, "app1", app) + } + { + throttled, app := throttler.IsAppThrottled("app2") + assert.False(t, throttled) + assert.Equal(t, "app2", app) + } + { + throttled, app := throttler.IsAppThrottled("app3") + assert.False(t, throttled) + assert.Equal(t, "app3", app) + } + { + throttled, app := throttler.IsAppThrottled("app4") + assert.False(t, throttled) + assert.Equal(t, "app4", app) + } assert.Equal(t, 0, throttler.throttledApps.ItemCount()) }) @@ -803,11 +819,31 @@ func TestIsAppThrottled(t *testing.T) { throttler.ThrottleApp("app2", time.Now(), DefaultThrottleRatio, false) // instantly expire throttler.ThrottleApp("app3", plusOneHour, DefaultThrottleRatio, false) throttler.ThrottleApp("app4", plusOneHour, 0, false) - assert.False(t, throttler.IsAppThrottled("app1")) // exempted - assert.False(t, throttler.IsAppThrottled("app2")) // expired - assert.True(t, throttler.IsAppThrottled("app3")) - assert.False(t, throttler.IsAppThrottled("app4")) // ratio is zero - assert.False(t, throttler.IsAppThrottled("app_other")) // not specified + { + throttled, app := throttler.IsAppThrottled("app1") + assert.False(t, throttled) // exempted + assert.Equal(t, "app1", app) + } + { + throttled, app := throttler.IsAppThrottled("app2") + assert.False(t, throttled) // expired + assert.Equal(t, "app2", app) + } + { + throttled, app := throttler.IsAppThrottled("app3") + assert.True(t, throttled) + assert.Equal(t, "app3", app) + } + { + throttled, app := throttler.IsAppThrottled("app4") + assert.False(t, throttled) // ratio is zero + assert.Equal(t, "app4", app) + } + { + throttled, app := throttler.IsAppThrottled("app_other") + assert.False(t, throttled) // not specified + assert.Equal(t, "app_other", app) + } assert.Equal(t, 3, throttler.throttledApps.ItemCount()) }) @@ -815,26 +851,67 @@ func TestIsAppThrottled(t *testing.T) { // throttle "all", see how it affects app throttler.ThrottleApp(throttlerapp.AllName.String(), plusOneHour, DefaultThrottleRatio, false) defer throttler.UnthrottleApp(throttlerapp.AllName.String()) - assert.True(t, throttler.IsAppThrottled("all")) // - assert.False(t, throttler.IsAppThrottled("app1")) // exempted - assert.True(t, throttler.IsAppThrottled("app2")) // expired, so falls under "all" - assert.True(t, throttler.IsAppThrottled("app3")) - assert.False(t, throttler.IsAppThrottled("app4")) // ratio is zero, there is a specific instruction for this app, so it doesn't fall under "all" - assert.True(t, throttler.IsAppThrottled("app_other")) // falls under "all" + { + throttled, app := throttler.IsAppThrottled("all") + assert.True(t, throttled) // explicitly throttled + assert.Equal(t, "all", app) + } + { + throttled, app := throttler.IsAppThrottled("app1") + assert.False(t, throttled) // exempted + assert.Equal(t, "app1", app) + } + { + throttled, app := throttler.IsAppThrottled("app2") + assert.True(t, throttled) // expired, so falls under "all" + assert.Equal(t, "all", app) + } + { + throttled, app := throttler.IsAppThrottled("app3") + assert.True(t, throttled) + assert.Equal(t, "app3", app) + } + { + throttled, app := throttler.IsAppThrottled("app4") + assert.False(t, throttled) // ratio is zero, there is a specific instruction for this app, so it doesn't fall under "all" + assert.Equal(t, "app4", app) + } + { + throttled, app := throttler.IsAppThrottled("app_other") + assert.True(t, throttled) // falls under "all" + assert.Equal(t, "all", app) + } // continuing previous test, we had 3 throttled apps. "all" is a new app being throttled. assert.Equal(t, 4, throttler.throttledApps.ItemCount()) }) - // + // // t.Run("unthrottle", func(t *testing.T) { throttler.UnthrottleApp("app1") throttler.UnthrottleApp("app2") throttler.UnthrottleApp("app3") throttler.UnthrottleApp("app4") - assert.False(t, throttler.IsAppThrottled("app1")) - assert.False(t, throttler.IsAppThrottled("app2")) - assert.False(t, throttler.IsAppThrottled("app3")) - assert.False(t, throttler.IsAppThrottled("app4")) + + { + throttled, app := throttler.IsAppThrottled("app1") + assert.False(t, throttled) + assert.Equal(t, "app1", app) + } + { + throttled, app := throttler.IsAppThrottled("app2") + assert.False(t, throttled) + assert.Equal(t, "app2", app) + } + { + throttled, app := throttler.IsAppThrottled("app3") + assert.False(t, throttled) + assert.Equal(t, "app3", app) + } + { + throttled, app := throttler.IsAppThrottled("app4") + assert.False(t, throttled) + assert.Equal(t, "app4", app) + } // we've manually unthrottled everything assert.Equal(t, 0, throttler.throttledApps.ItemCount()) @@ -843,12 +920,37 @@ func TestIsAppThrottled(t *testing.T) { // throttle "all", see how it affects app throttler.ThrottleApp(throttlerapp.AllName.String(), plusOneHour, DefaultThrottleRatio, false) defer throttler.UnthrottleApp(throttlerapp.AllName.String()) - assert.True(t, throttler.IsAppThrottled("all")) - assert.True(t, throttler.IsAppThrottled("app1")) - assert.True(t, throttler.IsAppThrottled("app2")) - assert.True(t, throttler.IsAppThrottled("app3")) - assert.True(t, throttler.IsAppThrottled("app4")) - assert.True(t, throttler.IsAppThrottled("app_other")) + + { + throttled, app := throttler.IsAppThrottled("all") + assert.True(t, throttled) // explicitly throttled + assert.Equal(t, "all", app) + } + { + throttled, app := throttler.IsAppThrottled("app1") + assert.True(t, throttled) + assert.Equal(t, "all", app) + } + { + throttled, app := throttler.IsAppThrottled("app2") + assert.True(t, throttled) + assert.Equal(t, "all", app) + } + { + throttled, app := throttler.IsAppThrottled("app3") + assert.True(t, throttled) + assert.Equal(t, "all", app) + } + { + throttled, app := throttler.IsAppThrottled("app4") + assert.True(t, throttled) + assert.Equal(t, "all", app) + } + { + throttled, app := throttler.IsAppThrottled("app_other") + assert.True(t, throttled) + assert.Equal(t, "all", app) + } // one rule, for "all" app assert.Equal(t, 1, throttler.throttledApps.ItemCount()) @@ -858,43 +960,120 @@ func TestIsAppThrottled(t *testing.T) { throttler.ThrottleApp("app3", plusOneHour, DefaultThrottleRatio, false) throttler.ThrottleApp(throttlerapp.AllName.String(), plusOneHour, DefaultThrottleRatio, true) defer throttler.UnthrottleApp(throttlerapp.AllName.String()) - assert.False(t, throttler.IsAppThrottled("all")) - assert.False(t, throttler.IsAppThrottled("app1")) - assert.False(t, throttler.IsAppThrottled("app2")) - assert.True(t, throttler.IsAppThrottled("app3")) - assert.False(t, throttler.IsAppThrottled("app4")) - assert.False(t, throttler.IsAppThrottled("app_other")) + { + throttled, app := throttler.IsAppThrottled("all") + assert.False(t, throttled) // explicitly throttled + assert.Equal(t, "all", app) + } + { + throttled, app := throttler.IsAppThrottled("app1") + assert.False(t, throttled) + assert.Equal(t, "app1", app) + } + { + throttled, app := throttler.IsAppThrottled("app2") + assert.False(t, throttled) + assert.Equal(t, "app2", app) + } + { + throttled, app := throttler.IsAppThrottled("app3") + assert.True(t, throttled) // explicitly throttled + assert.Equal(t, "app3", app) + } + { + throttled, app := throttler.IsAppThrottled("app4") + assert.False(t, throttled) + assert.Equal(t, "app4", app) + } + { + throttled, app := throttler.IsAppThrottled("app_other") + assert.False(t, throttled) + assert.Equal(t, "app_other", app) + } assert.Equal(t, 2, throttler.throttledApps.ItemCount()) }) } func TestIsAppExempted(t *testing.T) { - + plusOneHour := time.Now().Add(time.Hour) throttler := Throttler{ throttledApps: cache.New(cache.NoExpiration, 0), heartbeatWriter: &FakeHeartbeatWriter{}, } - assert.False(t, throttler.IsAppExempted("app1")) - assert.False(t, throttler.IsAppExempted("app2")) - assert.False(t, throttler.IsAppExempted("app3")) - // - throttler.ThrottleApp("app1", time.Now().Add(time.Hour), DefaultThrottleRatio, true) - throttler.ThrottleApp("app2", time.Now(), DefaultThrottleRatio, true) // instantly expire - assert.True(t, throttler.IsAppExempted("app1")) - assert.True(t, throttler.IsAppExempted("app1:other-tag")) - assert.False(t, throttler.IsAppExempted("app2")) // expired - assert.False(t, throttler.IsAppExempted("app3")) - // - throttler.UnthrottleApp("app1") - throttler.ThrottleApp("app2", time.Now().Add(time.Hour), DefaultThrottleRatio, false) - assert.False(t, throttler.IsAppExempted("app1")) - assert.False(t, throttler.IsAppExempted("app2")) - assert.False(t, throttler.IsAppExempted("app3")) - // - assert.True(t, throttler.IsAppExempted("schema-tracker")) - throttler.UnthrottleApp("schema-tracker") // meaningless. App is statically exempted - assert.True(t, throttler.IsAppExempted("schema-tracker")) + t.Run("initial", func(t *testing.T) { + { + exempted, app := throttler.IsAppExempted("app1") + assert.False(t, exempted) + assert.Equal(t, "app1", app) + } + { + exempted, app := throttler.IsAppExempted("app2") + assert.False(t, exempted) + assert.Equal(t, "app2", app) + } + { + exempted, app := throttler.IsAppExempted("app3") + assert.False(t, exempted) + assert.Equal(t, "app3", app) + } + }) + t.Run("exempt", func(t *testing.T) { + throttler.ThrottleApp("app1", time.Now().Add(time.Hour), DefaultThrottleRatio, true) + throttler.ThrottleApp("app2", time.Now(), DefaultThrottleRatio, true) // instantly expire + { + exempted, app := throttler.IsAppExempted("app1") + assert.True(t, exempted) + assert.Equal(t, "app1", app) + } + { + exempted, app := throttler.IsAppExempted("app1:other-tag") + assert.True(t, exempted) + assert.Equal(t, "app1", app) + } + { + exempted, app := throttler.IsAppExempted("app2") + assert.False(t, exempted) + assert.Equal(t, "app2", app) + } + { + exempted, app := throttler.IsAppExempted("app3") + assert.False(t, exempted) + assert.Equal(t, "app3", app) + } + }) + t.Run("throttle", func(t *testing.T) { + throttler.UnthrottleApp("app1") + throttler.ThrottleApp("app2", time.Now().Add(time.Hour), DefaultThrottleRatio, false) + { + exempted, app := throttler.IsAppExempted("app1") + assert.False(t, exempted) + assert.Equal(t, "app1", app) + } + { + exempted, app := throttler.IsAppExempted("app2") + assert.False(t, exempted) + assert.Equal(t, "app2", app) + } + { + exempted, app := throttler.IsAppExempted("app3") + assert.False(t, exempted) + assert.Equal(t, "app3", app) + } + }) + t.Run("special", func(t *testing.T) { + { + exempted, app := throttler.IsAppExempted("schema-tracker") + assert.True(t, exempted) + assert.Equal(t, "schema-tracker", app) + } + throttler.ThrottleApp("schema-tracker", plusOneHour, 1.0, false) // meaningless. App is statically exempted + { + exempted, app := throttler.IsAppExempted("schema-tracker") + assert.True(t, exempted) + assert.Equal(t, "schema-tracker", app) + } + }) } // TestRefreshInventory tests the behavior of the throttler's RefreshInventory() function, which @@ -1502,6 +1681,7 @@ func TestChecks(t *testing.T) { require.NotNil(t, checkResult) assert.EqualValues(t, 0.3, checkResult.Value) // self lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode) + assert.Equal(t, testAppName.String(), checkResult.AppName) assert.Len(t, checkResult.Metrics, 1) }) t.Run("explicit names", func(t *testing.T) { @@ -1513,6 +1693,7 @@ func TestChecks(t *testing.T) { t.Logf("%s: %+v", k, v) } } + assert.Equal(t, testAppName.String(), checkResult.AppName) assert.Equal(t, len(base.KnownMetricNames), len(checkResult.Metrics)) assert.EqualValues(t, 0.3, checkResult.Metrics[base.LagMetricName.String()].Value) // self lag value, because flags.Scope is set @@ -1533,6 +1714,7 @@ func TestChecks(t *testing.T) { t.Run("implicit names, always all known", func(t *testing.T) { checkResult := throttler.Check(ctx, throttlerapp.VitessName.String(), nil, flags) // "vitess" app always checks all known metrics: + assert.Equal(t, throttlerapp.VitessName.String(), checkResult.AppName) assert.Equal(t, len(base.KnownMetricNames), len(checkResult.Metrics)) }) t.Run("explicit names, irrelevant, always all known", func(t *testing.T) { @@ -1543,6 +1725,7 @@ func TestChecks(t *testing.T) { checkResult := throttler.Check(ctx, throttlerapp.VitessName.String(), metricNames, flags) require.NotNil(t, checkResult) + assert.Equal(t, throttlerapp.VitessName.String(), checkResult.AppName) assert.Equal(t, len(base.KnownMetricNames), len(checkResult.Metrics)) }) }) @@ -1558,6 +1741,7 @@ func TestChecks(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode) assert.ErrorIs(t, checkResult.Error, base.ErrThresholdExceeded) + assert.Equal(t, testAppName.String(), checkResult.AppName) assert.Len(t, checkResult.Metrics, 1) }) t.Run("explicit names", func(t *testing.T) { @@ -1566,6 +1750,7 @@ func TestChecks(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode) assert.ErrorIs(t, checkResult.Error, base.ErrThresholdExceeded) + assert.Equal(t, testAppName.String(), checkResult.AppName) assert.Equal(t, len(base.KnownMetricNames), len(checkResult.Metrics)) assert.EqualValues(t, 0.9, checkResult.Metrics[base.LagMetricName.String()].Value) // shard lag value, because flags.Scope is set From 090f96de7a06afc0e284fa17a8c5aab21941b7e3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:36:50 +0300 Subject: [PATCH 22/36] adding missing files Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../throttle/base/tablet_results.go | 99 +++++++++ .../throttle/base/tablet_results_test.go | 199 ++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 go/vt/vttablet/tabletserver/throttle/base/tablet_results.go create mode 100644 go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go diff --git a/go/vt/vttablet/tabletserver/throttle/base/tablet_results.go b/go/vt/vttablet/tabletserver/throttle/base/tablet_results.go new file mode 100644 index 00000000000..88e958e884e --- /dev/null +++ b/go/vt/vttablet/tabletserver/throttle/base/tablet_results.go @@ -0,0 +1,99 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package base + +import "sort" + +// TabletResultMap maps a tablet to a result +type TabletResultMap map[string]MetricResultMap + +func (m TabletResultMap) Split(alias string) (withAlias TabletResultMap, all TabletResultMap) { + withAlias = make(TabletResultMap) + if val, ok := m[alias]; ok { + withAlias[alias] = val + } + return withAlias, m +} + +func AggregateTabletMetricResults( + metricName MetricName, + tabletResultsMap TabletResultMap, + ignoreHostsCount int, + IgnoreDialTCPErrors bool, + ignoreHostsThreshold float64, +) (worstMetric MetricResult) { + // probes is known not to change. It can be *replaced*, but not changed. + // so it's safe to iterate it + probeValues := []float64{} + for _, tabletMetricResults := range tabletResultsMap { + tabletMetricResult, ok := tabletMetricResults[metricName] + if !ok { + return NoSuchMetric + } + if tabletMetricResult == nil { + return NoMetricResultYet + } + value, err := tabletMetricResult.Get() + if err != nil { + if IgnoreDialTCPErrors && IsDialTCPError(err) { + continue + } + if ignoreHostsCount > 0 { + // ok to skip this error + ignoreHostsCount = ignoreHostsCount - 1 + continue + } + return tabletMetricResult + } + + // No error + probeValues = append(probeValues, value) + } + if len(probeValues) == 0 { + return NoHostsMetricResult + } + + // If we got here, that means no errors (or good-to-skip errors) + sort.Float64s(probeValues) + // probeValues sorted ascending (from best, ie smallest, to worst, ie largest) + for ignoreHostsCount > 0 { + goodToIgnore := func() bool { + // Note that these hosts don't have errors + numProbeValues := len(probeValues) + if numProbeValues <= 1 { + // We wish to retain at least one host + return false + } + if ignoreHostsThreshold <= 0 { + // No threshold conditional (or implicitly "any value exceeds the threshold") + return true + } + if worstValue := probeValues[numProbeValues-1]; worstValue > ignoreHostsThreshold { + return true + } + return false + }() + if goodToIgnore { + probeValues = probeValues[0 : len(probeValues)-1] + } + // And, whether ignored or not, we are reducing our tokens + ignoreHostsCount = ignoreHostsCount - 1 + } + worstValue := probeValues[len(probeValues)-1] + worstMetric = NewSimpleMetricResult(worstValue) + return worstMetric +} diff --git a/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go b/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go new file mode 100644 index 00000000000..0cf953700e8 --- /dev/null +++ b/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go @@ -0,0 +1,199 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package base + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var ( + alias1 = "zone1-0001" + alias2 = "zone1-0002" + alias3 = "zone1-0003" + alias4 = "zone1-0004" + alias5 = "zone1-0005" +) + +const ( + nonexistentMetricName MetricName = "nonexistent" +) + +func newMetricResultMap(val float64) MetricResultMap { + return MetricResultMap{ + DefaultMetricName: NewSimpleMetricResult(val), + LagMetricName: NewSimpleMetricResult(val), + LoadAvgMetricName: NewSimpleMetricResult(3.14), + } +} +func noSuchMetricMap() MetricResultMap { + result := make(MetricResultMap) + for _, metricName := range KnownMetricNames { + result[metricName] = NoSuchMetric + } + return result +} + +func TestAggregateTabletMetricResultsNoErrors(t *testing.T) { + tabletResultsMap := TabletResultMap{ + alias1: newMetricResultMap(1.2), + alias2: newMetricResultMap(1.7), + alias3: newMetricResultMap(0.3), + alias4: newMetricResultMap(0.6), + alias5: newMetricResultMap(1.1), + } + + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 0, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.7) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 1, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.2) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 2, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.1) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 3, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.6) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 4, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.3) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 5, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.3) + } +} + +func TestAggregateTabletMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { + tabletResultsMap := TabletResultMap{ + alias1: newMetricResultMap(1.2), + alias2: newMetricResultMap(1.7), + alias3: newMetricResultMap(0.3), + alias4: newMetricResultMap(0.6), + alias5: newMetricResultMap(1.1), + } + + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 0, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.7) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 1, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.2) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 2, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.1) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 3, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.6) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 4, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.6) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 5, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.6) + } +} + +func TestAggregateTabletMetricResultsWithErrors(t *testing.T) { + tabletResultsMap := TabletResultMap{ + alias1: newMetricResultMap(1.2), + alias2: newMetricResultMap(1.7), + alias3: newMetricResultMap(0.3), + alias4: noSuchMetricMap(), + alias5: newMetricResultMap(1.1), + } + + t.Run("nonexistent", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(nonexistentMetricName, tabletResultsMap, 0, false, 0) + _, err := worstMetric.Get() + assert.Error(t, err) + assert.Equal(t, ErrNoSuchMetric, err) + }) + t.Run("no ignore", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 0, false, 0) + _, err := worstMetric.Get() + assert.Error(t, err) + assert.Equal(t, ErrNoSuchMetric, err) + }) + t.Run("ignore 1", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 1, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, 1.7, value) + }) + t.Run("ignore 2", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 2, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, 1.2, value) + }) + + tabletResultsMap[alias1][DefaultMetricName] = NoSuchMetric + t.Run("no such metric", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 0, false, 0) + _, err := worstMetric.Get() + assert.Error(t, err) + assert.Equal(t, ErrNoSuchMetric, err) + }) + t.Run("no such metric, ignore 1", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 1, false, 0) + _, err := worstMetric.Get() + assert.Error(t, err) + assert.Equal(t, ErrNoSuchMetric, err) + }) + t.Run("metric found", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 2, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.7) + }) +} From 8c204835af4da80afacf75b51fb7c43ab02e7d04 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:36:50 +0300 Subject: [PATCH 23/36] adding missing files Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../throttle/base/tablet_results.go | 99 +++++++++ .../throttle/base/tablet_results_test.go | 199 ++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 go/vt/vttablet/tabletserver/throttle/base/tablet_results.go create mode 100644 go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go diff --git a/go/vt/vttablet/tabletserver/throttle/base/tablet_results.go b/go/vt/vttablet/tabletserver/throttle/base/tablet_results.go new file mode 100644 index 00000000000..88e958e884e --- /dev/null +++ b/go/vt/vttablet/tabletserver/throttle/base/tablet_results.go @@ -0,0 +1,99 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package base + +import "sort" + +// TabletResultMap maps a tablet to a result +type TabletResultMap map[string]MetricResultMap + +func (m TabletResultMap) Split(alias string) (withAlias TabletResultMap, all TabletResultMap) { + withAlias = make(TabletResultMap) + if val, ok := m[alias]; ok { + withAlias[alias] = val + } + return withAlias, m +} + +func AggregateTabletMetricResults( + metricName MetricName, + tabletResultsMap TabletResultMap, + ignoreHostsCount int, + IgnoreDialTCPErrors bool, + ignoreHostsThreshold float64, +) (worstMetric MetricResult) { + // probes is known not to change. It can be *replaced*, but not changed. + // so it's safe to iterate it + probeValues := []float64{} + for _, tabletMetricResults := range tabletResultsMap { + tabletMetricResult, ok := tabletMetricResults[metricName] + if !ok { + return NoSuchMetric + } + if tabletMetricResult == nil { + return NoMetricResultYet + } + value, err := tabletMetricResult.Get() + if err != nil { + if IgnoreDialTCPErrors && IsDialTCPError(err) { + continue + } + if ignoreHostsCount > 0 { + // ok to skip this error + ignoreHostsCount = ignoreHostsCount - 1 + continue + } + return tabletMetricResult + } + + // No error + probeValues = append(probeValues, value) + } + if len(probeValues) == 0 { + return NoHostsMetricResult + } + + // If we got here, that means no errors (or good-to-skip errors) + sort.Float64s(probeValues) + // probeValues sorted ascending (from best, ie smallest, to worst, ie largest) + for ignoreHostsCount > 0 { + goodToIgnore := func() bool { + // Note that these hosts don't have errors + numProbeValues := len(probeValues) + if numProbeValues <= 1 { + // We wish to retain at least one host + return false + } + if ignoreHostsThreshold <= 0 { + // No threshold conditional (or implicitly "any value exceeds the threshold") + return true + } + if worstValue := probeValues[numProbeValues-1]; worstValue > ignoreHostsThreshold { + return true + } + return false + }() + if goodToIgnore { + probeValues = probeValues[0 : len(probeValues)-1] + } + // And, whether ignored or not, we are reducing our tokens + ignoreHostsCount = ignoreHostsCount - 1 + } + worstValue := probeValues[len(probeValues)-1] + worstMetric = NewSimpleMetricResult(worstValue) + return worstMetric +} diff --git a/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go b/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go new file mode 100644 index 00000000000..0cf953700e8 --- /dev/null +++ b/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go @@ -0,0 +1,199 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package base + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var ( + alias1 = "zone1-0001" + alias2 = "zone1-0002" + alias3 = "zone1-0003" + alias4 = "zone1-0004" + alias5 = "zone1-0005" +) + +const ( + nonexistentMetricName MetricName = "nonexistent" +) + +func newMetricResultMap(val float64) MetricResultMap { + return MetricResultMap{ + DefaultMetricName: NewSimpleMetricResult(val), + LagMetricName: NewSimpleMetricResult(val), + LoadAvgMetricName: NewSimpleMetricResult(3.14), + } +} +func noSuchMetricMap() MetricResultMap { + result := make(MetricResultMap) + for _, metricName := range KnownMetricNames { + result[metricName] = NoSuchMetric + } + return result +} + +func TestAggregateTabletMetricResultsNoErrors(t *testing.T) { + tabletResultsMap := TabletResultMap{ + alias1: newMetricResultMap(1.2), + alias2: newMetricResultMap(1.7), + alias3: newMetricResultMap(0.3), + alias4: newMetricResultMap(0.6), + alias5: newMetricResultMap(1.1), + } + + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 0, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.7) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 1, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.2) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 2, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.1) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 3, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.6) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 4, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.3) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 5, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.3) + } +} + +func TestAggregateTabletMetricResultsNoErrorsIgnoreHostsThreshold(t *testing.T) { + tabletResultsMap := TabletResultMap{ + alias1: newMetricResultMap(1.2), + alias2: newMetricResultMap(1.7), + alias3: newMetricResultMap(0.3), + alias4: newMetricResultMap(0.6), + alias5: newMetricResultMap(1.1), + } + + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 0, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.7) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 1, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.2) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 2, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.1) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 3, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.6) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 4, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.6) + } + { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 5, false, 1.0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 0.6) + } +} + +func TestAggregateTabletMetricResultsWithErrors(t *testing.T) { + tabletResultsMap := TabletResultMap{ + alias1: newMetricResultMap(1.2), + alias2: newMetricResultMap(1.7), + alias3: newMetricResultMap(0.3), + alias4: noSuchMetricMap(), + alias5: newMetricResultMap(1.1), + } + + t.Run("nonexistent", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(nonexistentMetricName, tabletResultsMap, 0, false, 0) + _, err := worstMetric.Get() + assert.Error(t, err) + assert.Equal(t, ErrNoSuchMetric, err) + }) + t.Run("no ignore", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 0, false, 0) + _, err := worstMetric.Get() + assert.Error(t, err) + assert.Equal(t, ErrNoSuchMetric, err) + }) + t.Run("ignore 1", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 1, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, 1.7, value) + }) + t.Run("ignore 2", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 2, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, 1.2, value) + }) + + tabletResultsMap[alias1][DefaultMetricName] = NoSuchMetric + t.Run("no such metric", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 0, false, 0) + _, err := worstMetric.Get() + assert.Error(t, err) + assert.Equal(t, ErrNoSuchMetric, err) + }) + t.Run("no such metric, ignore 1", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 1, false, 0) + _, err := worstMetric.Get() + assert.Error(t, err) + assert.Equal(t, ErrNoSuchMetric, err) + }) + t.Run("metric found", func(t *testing.T) { + worstMetric := AggregateTabletMetricResults(DefaultMetricName, tabletResultsMap, 2, false, 0) + value, err := worstMetric.Get() + assert.NoError(t, err) + assert.Equal(t, value, 1.7) + }) +} From a8eeea748a1c62baff9072436a172f35b3b7f523 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 15 Jul 2024 07:55:51 +0300 Subject: [PATCH 24/36] update copyright year Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../vttablet/tabletserver/throttle/base/tablet_results_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go b/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go index 0cf953700e8..98888eebbb8 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go +++ b/go/vt/vttablet/tabletserver/throttle/base/tablet_results_test.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Vitess Authors. +Copyright 2024 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From eb0635dbda5fe58f0e2e2710f4d1bc3e4b5acfbd Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:28:37 +0300 Subject: [PATCH 25/36] remove unused httpClient Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/throttler.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 3ef367d60b2..b84dc81a774 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -206,8 +206,6 @@ type Throttler struct { readSelfThrottleMetrics func(context.Context) base.ThrottleMetrics // overwritten by unit test - httpClient *http.Client - hostCpuCoreCount atomic.Int32 } @@ -263,7 +261,6 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv throttler.metricsHealth = cache.New(cache.NoExpiration, 0) throttler.appCheckedMetrics = cache.New(cache.NoExpiration, 0) - throttler.httpClient = base.SetupHTTPClient(2 * activeCollectInterval) throttler.initThrottleTabletTypes() throttler.check = NewThrottlerCheck(throttler) From 032c87a9239857fbaaee4c39ff6ca73385dab4d5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:35:34 +0300 Subject: [PATCH 26/36] adding Summary in CheckThrottlerResponse Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletmanagerdata/tabletmanagerdata.pb.go | 306 +++++++++--------- .../tabletmanagerdata_vtproto.pb.go | 44 +++ proto/tabletmanagerdata.proto | 3 + web/vtadmin/src/proto/vtadmin.d.ts | 6 + web/vtadmin/src/proto/vtadmin.js | 23 ++ 5 files changed, 234 insertions(+), 148 deletions(-) diff --git a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go index 133475bde34..715e2a2ab36 100644 --- a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go +++ b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go @@ -6638,6 +6638,8 @@ type CheckThrottlerResponse struct { Metrics map[string]*CheckThrottlerResponse_Metric `protobuf:"bytes,7,rep,name=metrics,proto3" json:"metrics,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // AppName is the name of app that was matched by the throttler AppName string `protobuf:"bytes,8,opt,name=app_name,json=appName,proto3" json:"app_name,omitempty"` + // Summary is a human readable analysis of the result + Summary string `protobuf:"bytes,9,opt,name=summary,proto3" json:"summary,omitempty"` } func (x *CheckThrottlerResponse) Reset() { @@ -6728,6 +6730,13 @@ func (x *CheckThrottlerResponse) GetAppName() string { return "" } +func (x *CheckThrottlerResponse) GetSummary() string { + if x != nil { + return x.Summary + } + return "" +} + type GetThrottlerStatusRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -8220,7 +8229,7 @@ var file_tabletmanagerdata_proto_rawDesc = []byte{ 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0xdd, 0x04, 0x0a, 0x16, + 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0xf7, 0x04, 0x0a, 0x16, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x74, 0x61, @@ -8240,161 +8249,162 @@ var file_tabletmanagerdata_proto_rawDesc = []byte{ 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x70, 0x4e, - 0x61, 0x6d, 0x65, 0x1a, 0xb7, 0x01, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, - 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x72, - 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x74, 0x68, - 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, - 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x1a, 0x6c, 0x0a, - 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x46, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, - 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, - 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1b, 0x0a, 0x19, 0x47, - 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xe1, 0x0f, 0x0a, 0x1a, 0x47, 0x65, 0x74, - 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, - 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, - 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1b, 0x0a, 0x09, - 0x69, 0x73, 0x5f, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, - 0x6f, 0x70, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4f, 0x70, - 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, - 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x64, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x74, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x44, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x74, - 0x12, 0x28, 0x0a, 0x10, 0x6c, 0x61, 0x67, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x61, 0x67, 0x4d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x2e, 0x0a, 0x13, 0x63, 0x75, - 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x71, 0x75, 0x65, 0x72, - 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x4d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x64, 0x65, - 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, - 0x0a, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x54, 0x68, - 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x3c, 0x0a, 0x1b, 0x6d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x61, 0x73, 0x5f, 0x64, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x17, 0x6d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x55, 0x73, 0x65, 0x64, 0x41, 0x73, 0x44, 0x65, - 0x66, 0x61, 0x75, 0x6c, 0x74, 0x12, 0x73, 0x0a, 0x12, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, - 0x74, 0x65, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x44, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, - 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, - 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x70, 0x0a, 0x11, 0x6d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x18, - 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, - 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, - 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x6d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x12, 0x67, 0x0a, 0x0e, - 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x18, 0x0e, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, - 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x67, 0x0a, 0x0e, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, - 0x65, 0x64, 0x5f, 0x61, 0x70, 0x70, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x68, 0x72, - 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x0d, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x12, 0x74, - 0x0a, 0x13, 0x61, 0x70, 0x70, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x6d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x74, 0x61, + 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x09, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x1a, 0xb7, 0x01, + 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x1a, 0x6c, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x46, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1b, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, + 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x22, 0xe1, 0x0f, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, + 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, + 0x6c, 0x69, 0x61, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x6c, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x6f, 0x70, 0x65, 0x6e, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x4f, 0x70, 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, + 0x69, 0x73, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x09, 0x69, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x69, + 0x73, 0x5f, 0x64, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x09, 0x69, 0x73, 0x44, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x6c, 0x61, + 0x67, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x61, 0x67, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x12, 0x2e, 0x0a, 0x13, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x6d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x11, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, + 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x10, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, + 0x64, 0x12, 0x3c, 0x0a, 0x1b, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x5f, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x61, 0x73, 0x5f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x17, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, + 0x6d, 0x65, 0x55, 0x73, 0x65, 0x64, 0x41, 0x73, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x12, + 0x73, 0x0a, 0x12, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x11, 0x61, 0x70, 0x70, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x6c, 0x79, - 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, - 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x12, - 0x5e, 0x0a, 0x0b, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x70, 0x70, 0x73, 0x18, 0x12, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, - 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x0a, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x73, 0x1a, - 0x3a, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x1a, 0x80, 0x01, 0x0a, 0x16, - 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x50, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, + 0x67, 0x61, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x11, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x12, 0x70, 0x0a, 0x11, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x74, + 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x43, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, + 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x12, 0x67, 0x0a, 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, + 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x0d, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, + 0x67, 0x0a, 0x0e, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x5f, 0x61, 0x70, 0x70, + 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x43, - 0x0a, 0x15, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, - 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x1a, 0x81, 0x01, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x48, 0x65, - 0x61, 0x6c, 0x74, 0x68, 0x12, 0x34, 0x0a, 0x0f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, - 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x0d, 0x6c, 0x61, 0x73, - 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x41, 0x74, 0x12, 0x3b, 0x0a, 0x1a, 0x73, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6c, 0x61, 0x73, 0x74, - 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x17, - 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x4c, 0x61, 0x73, 0x74, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x1a, 0x7c, 0x0a, 0x12, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x50, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, + 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x74, 0x68, 0x72, 0x6f, 0x74, + 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x12, 0x74, 0x0a, 0x13, 0x61, 0x70, 0x70, 0x5f, + 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, + 0x10, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, + 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x11, 0x61, 0x70, 0x70, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x29, + 0x0a, 0x10, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, + 0x65, 0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x72, 0x65, 0x63, 0x65, 0x6e, 0x74, + 0x6c, 0x79, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x12, 0x5e, 0x0a, 0x0b, 0x72, 0x65, 0x63, + 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x70, 0x70, 0x73, 0x18, 0x12, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5c, 0x0a, 0x12, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, - 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x74, - 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, - 0x64, 0x41, 0x70, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x1a, 0x44, 0x0a, 0x16, 0x41, 0x70, 0x70, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, - 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x59, 0x0a, 0x09, 0x52, 0x65, 0x63, - 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x12, 0x2b, 0x0a, 0x0a, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, - 0x64, 0x5f, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x74, 0x74, - 0x69, 0x6d, 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x09, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, - 0x64, 0x41, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, - 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x43, 0x6f, 0x64, 0x65, 0x1a, 0x76, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, - 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x4d, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, + 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x72, + 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x1a, 0x80, 0x01, 0x0a, 0x16, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, + 0x61, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x50, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x3a, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, + 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x43, 0x0a, 0x15, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x81, 0x01, + 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x34, + 0x0a, 0x0f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x61, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x79, 0x41, 0x74, 0x12, 0x3b, 0x0a, 0x1a, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x5f, + 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x17, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, + 0x73, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x79, 0x1a, 0x7c, 0x0a, 0x12, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x50, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, - 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, 0x3e, 0x0a, 0x19, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, - 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4e, 0x59, - 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x4f, 0x52, 0x44, 0x45, 0x52, 0x10, 0x01, 0x12, - 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x42, 0x30, 0x5a, 0x2e, - 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, - 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x48, 0x65, + 0x61, 0x6c, 0x74, 0x68, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, + 0x5c, 0x0a, 0x12, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x52, 0x75, + 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x44, 0x0a, + 0x16, 0x41, 0x70, 0x70, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x59, 0x0a, 0x09, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, + 0x12, 0x2b, 0x0a, 0x0a, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x52, 0x09, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1f, 0x0a, + 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x1a, 0x76, + 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x4d, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, + 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x52, 0x65, 0x63, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, 0x3e, 0x0a, 0x19, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, + 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4e, 0x59, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, + 0x49, 0x4e, 0x4f, 0x52, 0x44, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, + 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x42, 0x30, 0x5a, 0x2e, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, + 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/go/vt/proto/tabletmanagerdata/tabletmanagerdata_vtproto.pb.go b/go/vt/proto/tabletmanagerdata/tabletmanagerdata_vtproto.pb.go index 143519b063c..eb0b058a56e 100644 --- a/go/vt/proto/tabletmanagerdata/tabletmanagerdata_vtproto.pb.go +++ b/go/vt/proto/tabletmanagerdata/tabletmanagerdata_vtproto.pb.go @@ -2530,6 +2530,7 @@ func (m *CheckThrottlerResponse) CloneVT() *CheckThrottlerResponse { Message: m.Message, RecentlyChecked: m.RecentlyChecked, AppName: m.AppName, + Summary: m.Summary, } if rhs := m.Metrics; rhs != nil { tmpContainer := make(map[string]*CheckThrottlerResponse_Metric, len(rhs)) @@ -8834,6 +8835,13 @@ func (m *CheckThrottlerResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Summary) > 0 { + i -= len(m.Summary) + copy(dAtA[i:], m.Summary) + i = encodeVarint(dAtA, i, uint64(len(m.Summary))) + i-- + dAtA[i] = 0x4a + } if len(m.AppName) > 0 { i -= len(m.AppName) copy(dAtA[i:], m.AppName) @@ -11578,6 +11586,10 @@ func (m *CheckThrottlerResponse) SizeVT() (n int) { if l > 0 { n += 1 + l + sov(uint64(l)) } + l = len(m.Summary) + if l > 0 { + n += 1 + l + sov(uint64(l)) + } n += len(m.unknownFields) return n } @@ -25638,6 +25650,38 @@ func (m *CheckThrottlerResponse) UnmarshalVT(dAtA []byte) error { } m.AppName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Summary", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Summary = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skip(dAtA[iNdEx:]) diff --git a/proto/tabletmanagerdata.proto b/proto/tabletmanagerdata.proto index 76ce4b0b9ad..549c6c09782 100644 --- a/proto/tabletmanagerdata.proto +++ b/proto/tabletmanagerdata.proto @@ -774,6 +774,9 @@ message CheckThrottlerResponse { // AppName is the name of app that was matched by the throttler string app_name = 8; + + // Summary is a human readable analysis of the result + string summary = 9; } message GetThrottlerStatusRequest { diff --git a/web/vtadmin/src/proto/vtadmin.d.ts b/web/vtadmin/src/proto/vtadmin.d.ts index b9e0b22b14f..f12b795e444 100644 --- a/web/vtadmin/src/proto/vtadmin.d.ts +++ b/web/vtadmin/src/proto/vtadmin.d.ts @@ -30656,6 +30656,9 @@ export namespace tabletmanagerdata { /** CheckThrottlerResponse app_name */ app_name?: (string|null); + + /** CheckThrottlerResponse summary */ + summary?: (string|null); } /** Represents a CheckThrottlerResponse. */ @@ -30691,6 +30694,9 @@ export namespace tabletmanagerdata { /** CheckThrottlerResponse app_name. */ public app_name: string; + /** CheckThrottlerResponse summary. */ + public summary: string; + /** * Creates a new CheckThrottlerResponse instance using the specified properties. * @param [properties] Properties to set diff --git a/web/vtadmin/src/proto/vtadmin.js b/web/vtadmin/src/proto/vtadmin.js index ee8d4ec6262..5952dbed5b6 100644 --- a/web/vtadmin/src/proto/vtadmin.js +++ b/web/vtadmin/src/proto/vtadmin.js @@ -71284,6 +71284,7 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { * @property {boolean|null} [recently_checked] CheckThrottlerResponse recently_checked * @property {Object.|null} [metrics] CheckThrottlerResponse metrics * @property {string|null} [app_name] CheckThrottlerResponse app_name + * @property {string|null} [summary] CheckThrottlerResponse summary */ /** @@ -71366,6 +71367,14 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { */ CheckThrottlerResponse.prototype.app_name = ""; + /** + * CheckThrottlerResponse summary. + * @member {string} summary + * @memberof tabletmanagerdata.CheckThrottlerResponse + * @instance + */ + CheckThrottlerResponse.prototype.summary = ""; + /** * Creates a new CheckThrottlerResponse instance using the specified properties. * @function create @@ -71409,6 +71418,8 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { } if (message.app_name != null && Object.hasOwnProperty.call(message, "app_name")) writer.uint32(/* id 8, wireType 2 =*/66).string(message.app_name); + if (message.summary != null && Object.hasOwnProperty.call(message, "summary")) + writer.uint32(/* id 9, wireType 2 =*/74).string(message.summary); return writer; }; @@ -71494,6 +71505,10 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { message.app_name = reader.string(); break; } + case 9: { + message.summary = reader.string(); + break; + } default: reader.skipType(tag & 7); break; @@ -71560,6 +71575,9 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { if (message.app_name != null && message.hasOwnProperty("app_name")) if (!$util.isString(message.app_name)) return "app_name: string expected"; + if (message.summary != null && message.hasOwnProperty("summary")) + if (!$util.isString(message.summary)) + return "summary: string expected"; return null; }; @@ -71599,6 +71617,8 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { } if (object.app_name != null) message.app_name = String(object.app_name); + if (object.summary != null) + message.summary = String(object.summary); return message; }; @@ -71625,6 +71645,7 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { object.message = ""; object.recently_checked = false; object.app_name = ""; + object.summary = ""; } if (message.status_code != null && message.hasOwnProperty("status_code")) object.status_code = message.status_code; @@ -71646,6 +71667,8 @@ export const tabletmanagerdata = $root.tabletmanagerdata = (() => { } if (message.app_name != null && message.hasOwnProperty("app_name")) object.app_name = message.app_name; + if (message.summary != null && message.hasOwnProperty("summary")) + object.summary = message.summary; return object; }; From 35277455e2ef5f1fffe9120cf182f2040f9c3a2f Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:19:45 +0300 Subject: [PATCH 27/36] evaluate and populate metric result Summary Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletmanager/rpc_throttler.go | 2 ++ go/vt/vttablet/tabletserver/throttle/check.go | 12 ++++++----- .../tabletserver/throttle/check_result.go | 20 +++++++++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_throttler.go b/go/vt/vttablet/tabletmanager/rpc_throttler.go index 8ec3bb592da..5facbb01229 100644 --- a/go/vt/vttablet/tabletmanager/rpc_throttler.go +++ b/go/vt/vttablet/tabletmanager/rpc_throttler.go @@ -58,6 +58,8 @@ func (tm *TabletManager) CheckThrottler(ctx context.Context, req *tabletmanagerd Threshold: checkResult.Threshold, Message: checkResult.Message, RecentlyChecked: checkResult.RecentlyChecked, + AppName: checkResult.AppName, + Summary: checkResult.Summary(), Metrics: make(map[string]*tabletmanagerdatapb.CheckThrottlerResponse_Metric), } for name, metric := range checkResult.Metrics { diff --git a/go/vt/vttablet/tabletserver/throttle/check.go b/go/vt/vttablet/tabletserver/throttle/check.go index 8d61fddefdc..ea6139fe2d5 100644 --- a/go/vt/vttablet/tabletserver/throttle/check.go +++ b/go/vt/vttablet/tabletserver/throttle/check.go @@ -136,13 +136,14 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, scope ba metricNames = base.MetricNames{check.throttler.metricNameUsedAsDefault()} } metricNames = metricNames.Unique() - applyMetricToCheckResult := func(metric *MetricResult) { + applyMetricToCheckResult := func(metricName base.MetricName, metric *MetricResult) { checkResult.StatusCode = metric.StatusCode checkResult.Value = metric.Value checkResult.Threshold = metric.Threshold checkResult.Error = metric.Error checkResult.Message = metric.Message checkResult.AppName = metric.AppName + checkResult.MetricName = metricName.String() } for _, metricName := range metricNames { // Make sure not to modify the given scope. We create a new scope variable to work with. @@ -201,17 +202,18 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, scope ba // metrics, because a v20 primary would not know how to deal with it, and is not expecting any of those // metrics. // The only metric we ever report back is the default metric, see below. - applyMetricToCheckResult(metric) + applyMetricToCheckResult(metricName, metric) } } - if metric, ok := checkResult.Metrics[check.throttler.metricNameUsedAsDefault().String()]; ok && checkResult.IsOK() { - applyMetricToCheckResult(metric) + metricNameUsedAsDefault := check.throttler.metricNameUsedAsDefault() + if metric, ok := checkResult.Metrics[metricNameUsedAsDefault.String()]; ok && checkResult.IsOK() { + applyMetricToCheckResult(metricNameUsedAsDefault, metric) } if metric, ok := checkResult.Metrics[base.DefaultMetricName.String()]; ok && checkResult.IsOK() { // v20 compatibility: if this v21 server is a replica, reporting to a v20 primary, // then we must supply the v20-flavor check result. // If checkResult is not OK, then we will have populated these fields already by the failing metric. - applyMetricToCheckResult(metric) + applyMetricToCheckResult(base.DefaultMetricName, metric) } go func(statusCode int) { statsThrottlerCheckAnyTotal.Add(1) diff --git a/go/vt/vttablet/tabletserver/throttle/check_result.go b/go/vt/vttablet/tabletserver/throttle/check_result.go index 2025f205145..12007406fad 100644 --- a/go/vt/vttablet/tabletserver/throttle/check_result.go +++ b/go/vt/vttablet/tabletserver/throttle/check_result.go @@ -42,6 +42,7 @@ limitations under the License. package throttle import ( + "fmt" "net/http" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" @@ -66,6 +67,7 @@ type CheckResult struct { Message string `json:"Message"` RecentlyChecked bool `json:"RecentlyChecked"` AppName string `json:"AppName"` + MetricName string `json:"MetricName"` Metrics map[string]*MetricResult `json:"Metrics"` // New in multi-metrics support. Will eventually replace the above fields. } @@ -88,6 +90,24 @@ func (c *CheckResult) IsOK() bool { return c.StatusCode == http.StatusOK } +// Summary returns a human-readable summary of the check result +func (c *CheckResult) Summary() string { + switch c.StatusCode { + case http.StatusOK: + return fmt.Sprintf("%v is granted access", c.AppName) + case http.StatusExpectationFailed: + return fmt.Sprintf("%v is explicitly denied access", c.AppName) + case http.StatusInternalServerError: + return fmt.Sprintf("%v is denied access due to unexpected error: %v", c.AppName, c.Error) + case http.StatusTooManyRequests: + return fmt.Sprintf("%v is denied access due to %s metric value %v exceeding threshold %v", c.AppName, c.MetricName, c.Value, c.Threshold) + case http.StatusNotFound: + return fmt.Sprintf("%v is denied access due to unknown or uncollected metric", c.AppName) + default: + return fmt.Sprintf("unknown status code: %v", c.StatusCode) + } +} + // NewErrorCheckResult returns a check result that indicates an error func NewErrorCheckResult(statusCode int, err error) *CheckResult { return NewCheckResult(statusCode, 0, 0, "", err) From 32d1f3c6313796a41552cd181c66f90c5ce907dd Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 18 Jul 2024 13:06:54 +0300 Subject: [PATCH 28/36] apply matchedApp on indirect throttling. Add testing Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/check.go | 1 + .../tabletserver/throttle/check_result.go | 3 ++- .../tabletserver/throttle/throttler.go | 5 +++++ .../tabletserver/throttle/throttler_test.go | 21 +++++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/throttle/check.go b/go/vt/vttablet/tabletserver/throttle/check.go index ea6139fe2d5..460d27a5181 100644 --- a/go/vt/vttablet/tabletserver/throttle/check.go +++ b/go/vt/vttablet/tabletserver/throttle/check.go @@ -143,6 +143,7 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, scope ba checkResult.Error = metric.Error checkResult.Message = metric.Message checkResult.AppName = metric.AppName + checkResult.Scope = metric.Scope checkResult.MetricName = metricName.String() } for _, metricName := range metricNames { diff --git a/go/vt/vttablet/tabletserver/throttle/check_result.go b/go/vt/vttablet/tabletserver/throttle/check_result.go index 12007406fad..e51f1814d8a 100644 --- a/go/vt/vttablet/tabletserver/throttle/check_result.go +++ b/go/vt/vttablet/tabletserver/throttle/check_result.go @@ -68,6 +68,7 @@ type CheckResult struct { RecentlyChecked bool `json:"RecentlyChecked"` AppName string `json:"AppName"` MetricName string `json:"MetricName"` + Scope string `json:"Scope"` Metrics map[string]*MetricResult `json:"Metrics"` // New in multi-metrics support. Will eventually replace the above fields. } @@ -100,7 +101,7 @@ func (c *CheckResult) Summary() string { case http.StatusInternalServerError: return fmt.Sprintf("%v is denied access due to unexpected error: %v", c.AppName, c.Error) case http.StatusTooManyRequests: - return fmt.Sprintf("%v is denied access due to %s metric value %v exceeding threshold %v", c.AppName, c.MetricName, c.Value, c.Threshold) + return fmt.Sprintf("%v is denied access due to %s/%s metric value %v exceeding threshold %v", c.AppName, c.Scope, c.MetricName, c.Value, c.Threshold) case http.StatusNotFound: return fmt.Sprintf("%v is denied access due to unknown or uncollected metric", c.AppName) default: diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index b84dc81a774..01c4fb3c622 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1541,6 +1541,7 @@ func (throttler *Throttler) checkScope(ctx context.Context, appName string, scop return result } + matchedApp := appName if len(metricNames) == 0 { // No explicit metrics requested. // Get the metric names mappd to the given app @@ -1554,6 +1555,7 @@ func (throttler *Throttler) checkScope(ctx context.Context, appName string, scop case []base.MetricName: metricNames = append(metricNames, val...) } + matchedApp = appToken } } } @@ -1567,17 +1569,20 @@ func (throttler *Throttler) checkScope(ctx context.Context, appName string, scop case []base.MetricName: metricNames = val } + matchedApp = throttlerapp.AllName.String() } } if throttlerapp.VitessName.Equals(appName) { // "vitess" always checks all metrics, irrespective of what is mapped. metricNames = base.KnownMetricNames + matchedApp = appName } if len(metricNames) == 0 { // Nothing mapped? For backwards compatibility and as default, we use the "default" metric. metricNames = base.MetricNames{throttler.metricNameUsedAsDefault()} } checkResult = throttler.check.Check(ctx, appName, scope, metricNames, flags) + checkResult.AppName = matchedApp shouldRequestHeartbeats := !flags.SkipRequestHeartbeats if throttlerapp.VitessName.Equals(appName) { diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index accea016b1e..1ed0ce3f126 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -418,6 +418,7 @@ func TestApplyThrottlerConfigMetricThresholds(t *testing.T) { assert.EqualValues(t, 0.3, checkResult.Value) // self lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) t.Run("apply low threshold", func(t *testing.T) { assert.Equal(t, 0.75, throttler.GetMetricsThreshold()) @@ -440,6 +441,7 @@ func TestApplyThrottlerConfigMetricThresholds(t *testing.T) { assert.EqualValues(t, 0.3, checkResult.Value, "unexpected result: %+v", checkResult) // self lag value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "test is denied access due to self/lag metric value") }) t.Run("apply low threshold but high 'lag' override", func(t *testing.T) { throttlerConfig := &topodatapb.ThrottlerConfig{ @@ -463,6 +465,7 @@ func TestApplyThrottlerConfigMetricThresholds(t *testing.T) { assert.EqualValues(t, 0.3, checkResult.Value, "unexpected result: %+v", checkResult) // self lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) }) @@ -520,6 +523,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "test is denied access due to shard/lag metric value") }) t.Run("apply high lag threshold", func(t *testing.T) { throttlerConfig.Threshold = 4444.0 @@ -534,6 +538,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // self lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) }) t.Run("apply low 'loadavg' threshold", func(t *testing.T) { @@ -548,6 +553,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) }) t.Run("assign 'loadavg' to test app", func(t *testing.T) { @@ -566,6 +572,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // self loadavg value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "test is denied access due to self/loadavg metric value") }) }) t.Run("assign 'shard/loadavg' to test app", func(t *testing.T) { @@ -584,6 +591,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 5.1, checkResult.Value) // shard loadavg value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "test is denied access due to shard/loadavg metric value") }) }) t.Run("assign 'lag,loadavg' to test app", func(t *testing.T) { @@ -601,6 +609,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // self loadavg value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) + assert.Contains(t, checkResult.Summary(), "test is denied access due to self/loadavg metric value") }) }) t.Run("assign 'lag,shard/loadavg' to test app", func(t *testing.T) { @@ -618,6 +627,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 5.1, checkResult.Value) // shard loadavg value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) + assert.Contains(t, checkResult.Summary(), "test is denied access due to shard/loadavg metric value") }) }) t.Run("clear 'loadavg' threshold", func(t *testing.T) { @@ -631,6 +641,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 1, len(checkResult.Metrics), "unexpected metrics: %+v", checkResult.Metrics) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) }) t.Run("assign 'lag,threads_running' to test app", func(t *testing.T) { @@ -648,6 +659,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) }) t.Run("assign 'custom,loadavg' to 'all' app", func(t *testing.T) { @@ -665,6 +677,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // loadavg self value exceeds threshold assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) + assert.Contains(t, checkResult.Summary(), "all is denied access due to self/loadavg metric value") }) t.Run("check 'test' after assignment", func(t *testing.T) { // "test" app unaffected by 'all' assignment, because it has @@ -679,6 +692,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) t.Run("'online-ddl' app affected by 'all'", func(t *testing.T) { // "online-ddl" app is affected by 'all' assignment, because it has @@ -692,6 +706,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // loadavg self value exceeds threshold assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) + assert.Contains(t, checkResult.Summary(), "all is denied access due to self/loadavg metric value") }) }) t.Run("'vreplication:online-ddl:12345' app affected by 'all'", func(t *testing.T) { @@ -702,6 +717,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // loadavg self value exceeds threshold assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) + assert.Contains(t, checkResult.Summary(), "all is denied access due to self/loadavg metric value") }) t.Run("'vreplication:online-ddl:test' app affected by 'test' and not by 'all'", func(t *testing.T) { // "vreplication:online-ddl:test" app is affected by 'test' assignment, because it has @@ -711,6 +727,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) t.Run("deassign metrics from 'all' app", func(t *testing.T) { delete(throttlerConfig.AppCheckedMetrics, throttlerapp.AllName.String()) @@ -725,6 +742,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "all is granted access") }) t.Run("check 'test' after assignment", func(t *testing.T) { // "test" app unaffected by the entire 'all' assignment, because it has @@ -739,6 +757,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) t.Run("'online-ddl' no longer has 'all' impact", func(t *testing.T) { // "online-ddl" app is affected by 'all' assignment, because it has @@ -752,6 +771,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "online-ddl is granted access") }) }) @@ -768,6 +788,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) + assert.Contains(t, checkResult.Summary(), "test is granted access") }) }) From 0f48902187ea75921d2383a8682728a1f8e1b330 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:24:24 +0300 Subject: [PATCH 29/36] proto: add ThrottledReason in VStreamRowsResponse and in VEvent Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/proto/binlogdata/binlogdata.pb.go | 374 ++++++++++-------- .../proto/binlogdata/binlogdata_vtproto.pb.go | 126 +++++- proto/binlogdata.proto | 4 + web/vtadmin/src/proto/vtadmin.d.ts | 12 + web/vtadmin/src/proto/vtadmin.js | 46 +++ 5 files changed, 369 insertions(+), 193 deletions(-) diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index aade1d049f8..8374a4a2733 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -1913,6 +1913,8 @@ type VEvent struct { Shard string `protobuf:"bytes,23,opt,name=shard,proto3" json:"shard,omitempty"` // indicate that we are being throttled right now Throttled bool `protobuf:"varint,24,opt,name=throttled,proto3" json:"throttled,omitempty"` + // ThrottledReason is a human readable string that explains why the stream is throttled + ThrottledReason string `protobuf:"bytes,25,opt,name=throttled_reason,json=throttledReason,proto3" json:"throttled_reason,omitempty"` } func (x *VEvent) Reset() { @@ -2045,6 +2047,13 @@ func (x *VEvent) GetThrottled() bool { return false } +func (x *VEvent) GetThrottledReason() string { + if x != nil { + return x.ThrottledReason + } + return "" +} + type MinimalTable struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2397,6 +2406,8 @@ type VStreamRowsResponse struct { Throttled bool `protobuf:"varint,6,opt,name=throttled,proto3" json:"throttled,omitempty"` // Heartbeat indicates that this is a heartbeat message Heartbeat bool `protobuf:"varint,7,opt,name=heartbeat,proto3" json:"heartbeat,omitempty"` + // ThrottledReason is a human readable string that explains why the stream is throttled + ThrottledReason string `protobuf:"bytes,8,opt,name=throttled_reason,json=throttledReason,proto3" json:"throttled_reason,omitempty"` } func (x *VStreamRowsResponse) Reset() { @@ -2480,6 +2491,13 @@ func (x *VStreamRowsResponse) GetHeartbeat() bool { return false } +func (x *VStreamRowsResponse) GetThrottledReason() string { + if x != nil { + return x.ThrottledReason + } + return "" +} + // VStreamTablesRequest is the payload for VStreamTables type VStreamTablesRequest struct { state protoimpl.MessageState @@ -3247,7 +3265,7 @@ var file_binlogdata_proto_rawDesc = []byte{ 0x68, 0x61, 0x72, 0x64, 0x52, 0x0c, 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x22, 0x8b, 0x04, + 0x75, 0x72, 0x63, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x22, 0xb6, 0x04, 0x0a, 0x06, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, @@ -3280,183 +3298,189 @@ var file_binlogdata_proto_rawDesc = []byte{ 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x17, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x18, 0x18, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x09, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x22, 0x8d, 0x01, 0x0a, 0x0c, - 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, - 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x70, 0x5f, 0x6b, 0x5f, 0x63, 0x6f, - 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, 0x09, 0x70, 0x4b, 0x43, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0e, 0x70, 0x5f, 0x6b, 0x5f, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x70, 0x4b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x41, 0x0a, 0x0d, 0x4d, - 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x30, 0x0a, 0x06, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x62, - 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, - 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x22, 0xc7, - 0x02, 0x0a, 0x0e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, - 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, - 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, - 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, - 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, - 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, - 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, - 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, - 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x06, - 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x62, - 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x3e, 0x0a, 0x0f, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x5f, 0x6b, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, + 0x52, 0x09, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x74, + 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, + 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, + 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x8d, 0x01, 0x0a, 0x0c, 0x4d, 0x69, 0x6e, 0x69, 0x6d, + 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, + 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x70, 0x5f, 0x6b, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, 0x09, 0x70, 0x4b, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, + 0x73, 0x12, 0x23, 0x0a, 0x0e, 0x70, 0x5f, 0x6b, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x4b, 0x49, 0x6e, 0x64, + 0x65, 0x78, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x41, 0x0a, 0x0d, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, + 0x6c, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x30, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x22, 0xc7, 0x02, 0x0a, 0x0e, 0x56, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x13, + 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, 0x70, + 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, 0x65, + 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, 0x0a, + 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, + 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, + 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, + 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x12, 0x3e, 0x0a, 0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6c, 0x61, 0x73, + 0x74, 0x5f, 0x70, 0x5f, 0x6b, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x62, + 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4c, + 0x61, 0x73, 0x74, 0x50, 0x4b, 0x52, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, + 0x50, 0x4b, 0x73, 0x22, 0x3d, 0x0a, 0x0f, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x22, 0x85, 0x02, 0x0a, 0x12, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x6f, + 0x77, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, + 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, + 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, + 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, + 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, + 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, + 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x2a, + 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x22, 0xa4, 0x02, 0x0a, 0x13, 0x56, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x08, 0x70, 0x6b, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x08, 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, + 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x22, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, + 0x6f, 0x77, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, + 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, + 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x68, 0x65, 0x61, 0x72, + 0x74, 0x62, 0x65, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x68, 0x65, 0x61, + 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, + 0x6c, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0f, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, + 0x6e, 0x22, 0xc5, 0x01, 0x0a, 0x14, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, + 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, + 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, + 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, + 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, + 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xde, 0x01, 0x0a, 0x15, 0x56, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x08, 0x70, 0x6b, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x08, 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x05, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, + 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x22, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, + 0x6f, 0x77, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x22, 0x69, 0x0a, 0x0b, 0x4c, 0x61, + 0x73, 0x74, 0x50, 0x4b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3c, 0x0a, 0x0e, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x5f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, - 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x52, 0x0c, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x73, 0x22, 0x3d, 0x0a, 0x0f, 0x56, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x62, 0x69, - 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, - 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x85, 0x02, 0x0a, 0x12, 0x56, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, - 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, - 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, - 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, - 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, - 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, - 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, - 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, - 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, - 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, - 0x65, 0x72, 0x79, 0x12, 0x2a, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x18, 0x05, 0x20, + 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x58, 0x0a, 0x0b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, + 0x73, 0x74, 0x50, 0x4b, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x22, - 0xf9, 0x01, 0x0a, 0x13, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, - 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x28, 0x0a, - 0x08, 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x08, 0x70, - 0x6b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, 0x72, - 0x6f, 0x77, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, - 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x22, 0x0a, 0x06, 0x6c, - 0x61, 0x73, 0x74, 0x70, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, - 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x12, - 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x09, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, - 0x09, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x09, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x22, 0xc5, 0x01, 0x0a, 0x14, - 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, - 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, - 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, - 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, - 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, - 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, - 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x22, 0xde, 0x01, 0x0a, 0x15, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, - 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, - 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x06, - 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x12, 0x28, 0x0a, 0x08, 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x52, 0x08, 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x12, 0x0a, 0x04, - 0x67, 0x74, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, - 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, - 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, - 0x12, 0x22, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x06, 0x6c, 0x61, - 0x73, 0x74, 0x70, 0x6b, 0x22, 0x69, 0x0a, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x12, 0x3c, 0x0a, 0x0e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6c, 0x61, 0x73, - 0x74, 0x5f, 0x70, 0x5f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x62, 0x69, - 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, - 0x73, 0x74, 0x50, 0x4b, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, - 0x4b, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, - 0x58, 0x0a, 0x0b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x12, 0x1d, - 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, - 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x22, 0xdc, 0x01, 0x0a, 0x15, 0x56, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, - 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, - 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, - 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, - 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, - 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, - 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, - 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x72, 0x0a, 0x16, 0x56, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, - 0x72, 0x6f, 0x77, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, - 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x2a, 0x3e, 0x0a, 0x0b, - 0x4f, 0x6e, 0x44, 0x44, 0x4c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x49, - 0x47, 0x4e, 0x4f, 0x52, 0x45, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x54, 0x4f, 0x50, 0x10, - 0x01, 0x12, 0x08, 0x0a, 0x04, 0x45, 0x58, 0x45, 0x43, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x45, - 0x58, 0x45, 0x43, 0x5f, 0x49, 0x47, 0x4e, 0x4f, 0x52, 0x45, 0x10, 0x03, 0x2a, 0x7b, 0x0a, 0x18, - 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x4d, 0x61, 0x74, 0x65, - 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x4d, 0x6f, 0x76, - 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x02, - 0x12, 0x0b, 0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x10, 0x03, 0x12, 0x0b, 0x0a, - 0x07, 0x52, 0x65, 0x73, 0x68, 0x61, 0x72, 0x64, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x4f, 0x6e, - 0x6c, 0x69, 0x6e, 0x65, 0x44, 0x44, 0x4c, 0x10, 0x05, 0x2a, 0x44, 0x0a, 0x1b, 0x56, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, - 0x77, 0x53, 0x75, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, - 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x10, 0x01, 0x12, - 0x0e, 0x0a, 0x0a, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x43, 0x6f, 0x70, 0x79, 0x10, 0x02, 0x2a, - 0x71, 0x0a, 0x19, 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, - 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, - 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x6e, 0x69, - 0x74, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x74, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x10, 0x02, - 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x6f, 0x70, 0x79, 0x69, 0x6e, 0x67, 0x10, 0x03, 0x12, 0x0b, 0x0a, - 0x07, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x72, - 0x72, 0x6f, 0x72, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x4c, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, - 0x10, 0x06, 0x2a, 0x8d, 0x02, 0x0a, 0x0a, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x08, - 0x0a, 0x04, 0x47, 0x54, 0x49, 0x44, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x45, 0x47, 0x49, - 0x4e, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x03, 0x12, - 0x0c, 0x0a, 0x08, 0x52, 0x4f, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, 0x12, 0x07, 0x0a, - 0x03, 0x44, 0x44, 0x4c, 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x4e, 0x53, 0x45, 0x52, 0x54, - 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, 0x07, 0x12, - 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x08, 0x12, 0x0a, 0x0a, 0x06, 0x44, - 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x09, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x45, 0x54, 0x10, 0x0a, - 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x54, 0x48, 0x45, 0x52, 0x10, 0x0b, 0x12, 0x07, 0x0a, 0x03, 0x52, - 0x4f, 0x57, 0x10, 0x0c, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x10, 0x0d, 0x12, - 0x0d, 0x0a, 0x09, 0x48, 0x45, 0x41, 0x52, 0x54, 0x42, 0x45, 0x41, 0x54, 0x10, 0x0e, 0x12, 0x09, - 0x0a, 0x05, 0x56, 0x47, 0x54, 0x49, 0x44, 0x10, 0x0f, 0x12, 0x0b, 0x0a, 0x07, 0x4a, 0x4f, 0x55, - 0x52, 0x4e, 0x41, 0x4c, 0x10, 0x10, 0x12, 0x0b, 0x0a, 0x07, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, - 0x4e, 0x10, 0x11, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x53, 0x54, 0x50, 0x4b, 0x10, 0x12, 0x12, - 0x0d, 0x0a, 0x09, 0x53, 0x41, 0x56, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x13, 0x12, 0x12, - 0x0a, 0x0e, 0x43, 0x4f, 0x50, 0x59, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, - 0x10, 0x14, 0x2a, 0x27, 0x0a, 0x0d, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x53, 0x10, 0x00, 0x12, - 0x0a, 0x0a, 0x06, 0x53, 0x48, 0x41, 0x52, 0x44, 0x53, 0x10, 0x01, 0x42, 0x29, 0x5a, 0x27, 0x76, - 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, - 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x62, 0x69, 0x6e, 0x6c, - 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0xdc, 0x01, 0x0a, 0x15, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, + 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, + 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, + 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, + 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, + 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, + 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x72, + 0x0a, 0x16, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x12, + 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, + 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, + 0x77, 0x73, 0x2a, 0x3e, 0x0a, 0x0b, 0x4f, 0x6e, 0x44, 0x44, 0x4c, 0x41, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x47, 0x4e, 0x4f, 0x52, 0x45, 0x10, 0x00, 0x12, 0x08, 0x0a, + 0x04, 0x53, 0x54, 0x4f, 0x50, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x45, 0x58, 0x45, 0x43, 0x10, + 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x58, 0x45, 0x43, 0x5f, 0x49, 0x47, 0x4e, 0x4f, 0x52, 0x45, + 0x10, 0x03, 0x2a, 0x7b, 0x0a, 0x18, 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, + 0x0a, 0x0b, 0x4d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x10, 0x00, 0x12, + 0x0e, 0x0a, 0x0a, 0x4d, 0x6f, 0x76, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x10, 0x01, 0x12, + 0x15, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x65, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x68, 0x61, 0x72, 0x64, 0x10, 0x04, + 0x12, 0x0d, 0x0a, 0x09, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x44, 0x44, 0x4c, 0x10, 0x05, 0x2a, + 0x44, 0x0a, 0x1b, 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x75, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, + 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x74, + 0x69, 0x61, 0x6c, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x43, + 0x6f, 0x70, 0x79, 0x10, 0x02, 0x2a, 0x71, 0x0a, 0x19, 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, + 0x08, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x74, 0x6f, + 0x70, 0x70, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x6f, 0x70, 0x79, 0x69, 0x6e, + 0x67, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x04, + 0x12, 0x09, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x4c, + 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x10, 0x06, 0x2a, 0x8d, 0x02, 0x0a, 0x0a, 0x56, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, + 0x57, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x47, 0x54, 0x49, 0x44, 0x10, 0x01, 0x12, 0x09, + 0x0a, 0x05, 0x42, 0x45, 0x47, 0x49, 0x4e, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4d, + 0x4d, 0x49, 0x54, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x4f, 0x4c, 0x4c, 0x42, 0x41, 0x43, + 0x4b, 0x10, 0x04, 0x12, 0x07, 0x0a, 0x03, 0x44, 0x44, 0x4c, 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06, + 0x49, 0x4e, 0x53, 0x45, 0x52, 0x54, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x50, 0x4c, + 0x41, 0x43, 0x45, 0x10, 0x07, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, + 0x08, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x09, 0x12, 0x07, 0x0a, + 0x03, 0x53, 0x45, 0x54, 0x10, 0x0a, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x54, 0x48, 0x45, 0x52, 0x10, + 0x0b, 0x12, 0x07, 0x0a, 0x03, 0x52, 0x4f, 0x57, 0x10, 0x0c, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x49, + 0x45, 0x4c, 0x44, 0x10, 0x0d, 0x12, 0x0d, 0x0a, 0x09, 0x48, 0x45, 0x41, 0x52, 0x54, 0x42, 0x45, + 0x41, 0x54, 0x10, 0x0e, 0x12, 0x09, 0x0a, 0x05, 0x56, 0x47, 0x54, 0x49, 0x44, 0x10, 0x0f, 0x12, + 0x0b, 0x0a, 0x07, 0x4a, 0x4f, 0x55, 0x52, 0x4e, 0x41, 0x4c, 0x10, 0x10, 0x12, 0x0b, 0x0a, 0x07, + 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x11, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x53, + 0x54, 0x50, 0x4b, 0x10, 0x12, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x41, 0x56, 0x45, 0x50, 0x4f, 0x49, + 0x4e, 0x54, 0x10, 0x13, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x4f, 0x50, 0x59, 0x5f, 0x43, 0x4f, 0x4d, + 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x14, 0x2a, 0x27, 0x0a, 0x0d, 0x4d, 0x69, 0x67, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x54, 0x41, 0x42, + 0x4c, 0x45, 0x53, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x48, 0x41, 0x52, 0x44, 0x53, 0x10, + 0x01, 0x42, 0x29, 0x5a, 0x27, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, + 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go b/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go index 1332681a976..ea15c40a992 100644 --- a/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go +++ b/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go @@ -512,20 +512,21 @@ func (m *VEvent) CloneVT() *VEvent { return (*VEvent)(nil) } r := &VEvent{ - Type: m.Type, - Timestamp: m.Timestamp, - Gtid: m.Gtid, - Statement: m.Statement, - RowEvent: m.RowEvent.CloneVT(), - FieldEvent: m.FieldEvent.CloneVT(), - Vgtid: m.Vgtid.CloneVT(), - Journal: m.Journal.CloneVT(), - Dml: m.Dml, - CurrentTime: m.CurrentTime, - LastPKEvent: m.LastPKEvent.CloneVT(), - Keyspace: m.Keyspace, - Shard: m.Shard, - Throttled: m.Throttled, + Type: m.Type, + Timestamp: m.Timestamp, + Gtid: m.Gtid, + Statement: m.Statement, + RowEvent: m.RowEvent.CloneVT(), + FieldEvent: m.FieldEvent.CloneVT(), + Vgtid: m.Vgtid.CloneVT(), + Journal: m.Journal.CloneVT(), + Dml: m.Dml, + CurrentTime: m.CurrentTime, + LastPKEvent: m.LastPKEvent.CloneVT(), + Keyspace: m.Keyspace, + Shard: m.Shard, + Throttled: m.Throttled, + ThrottledReason: m.ThrottledReason, } if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) @@ -671,10 +672,11 @@ func (m *VStreamRowsResponse) CloneVT() *VStreamRowsResponse { return (*VStreamRowsResponse)(nil) } r := &VStreamRowsResponse{ - Gtid: m.Gtid, - Lastpk: m.Lastpk.CloneVT(), - Throttled: m.Throttled, - Heartbeat: m.Heartbeat, + Gtid: m.Gtid, + Lastpk: m.Lastpk.CloneVT(), + Throttled: m.Throttled, + Heartbeat: m.Heartbeat, + ThrottledReason: m.ThrottledReason, } if rhs := m.Fields; rhs != nil { tmpContainer := make([]*query.Field, len(rhs)) @@ -2131,6 +2133,15 @@ func (m *VEvent) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.ThrottledReason) > 0 { + i -= len(m.ThrottledReason) + copy(dAtA[i:], m.ThrottledReason) + i = encodeVarint(dAtA, i, uint64(len(m.ThrottledReason))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xca + } if m.Throttled { i-- if m.Throttled { @@ -2626,6 +2637,13 @@ func (m *VStreamRowsResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.ThrottledReason) > 0 { + i -= len(m.ThrottledReason) + copy(dAtA[i:], m.ThrottledReason) + i = encodeVarint(dAtA, i, uint64(len(m.ThrottledReason))) + i-- + dAtA[i] = 0x42 + } if m.Heartbeat { i-- if m.Heartbeat { @@ -3739,6 +3757,10 @@ func (m *VEvent) SizeVT() (n int) { if m.Throttled { n += 3 } + l = len(m.ThrottledReason) + if l > 0 { + n += 2 + l + sov(uint64(l)) + } n += len(m.unknownFields) return n } @@ -3910,6 +3932,10 @@ func (m *VStreamRowsResponse) SizeVT() (n int) { if m.Heartbeat { n += 2 } + l = len(m.ThrottledReason) + if l > 0 { + n += 1 + l + sov(uint64(l)) + } n += len(m.unknownFields) return n } @@ -7951,6 +7977,38 @@ func (m *VEvent) UnmarshalVT(dAtA []byte) error { } } m.Throttled = bool(v != 0) + case 25: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ThrottledReason", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ThrottledReason = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skip(dAtA[iNdEx:]) @@ -9116,6 +9174,38 @@ func (m *VStreamRowsResponse) UnmarshalVT(dAtA []byte) error { } } m.Heartbeat = bool(v != 0) + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ThrottledReason", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ThrottledReason = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skip(dAtA[iNdEx:]) diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index 1e70275e8b5..5f5bbd59c6e 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -458,6 +458,8 @@ message VEvent { string shard = 23; // indicate that we are being throttled right now bool throttled = 24; + // ThrottledReason is a human readable string that explains why the stream is throttled + string throttled_reason = 25; } message MinimalTable { @@ -511,6 +513,8 @@ message VStreamRowsResponse { bool throttled = 6; // Heartbeat indicates that this is a heartbeat message bool heartbeat = 7; + // ThrottledReason is a human readable string that explains why the stream is throttled + string throttled_reason = 8; } diff --git a/web/vtadmin/src/proto/vtadmin.d.ts b/web/vtadmin/src/proto/vtadmin.d.ts index f12b795e444..5237974ea18 100644 --- a/web/vtadmin/src/proto/vtadmin.d.ts +++ b/web/vtadmin/src/proto/vtadmin.d.ts @@ -33821,6 +33821,9 @@ export namespace binlogdata { /** VEvent throttled */ throttled?: (boolean|null); + + /** VEvent throttled_reason */ + throttled_reason?: (string|null); } /** Represents a VEvent. */ @@ -33874,6 +33877,9 @@ export namespace binlogdata { /** VEvent throttled. */ public throttled: boolean; + /** VEvent throttled_reason. */ + public throttled_reason: string; + /** * Creates a new VEvent instance using the specified properties. * @param [properties] Properties to set @@ -34532,6 +34538,9 @@ export namespace binlogdata { /** VStreamRowsResponse heartbeat */ heartbeat?: (boolean|null); + + /** VStreamRowsResponse throttled_reason */ + throttled_reason?: (string|null); } /** Represents a VStreamRowsResponse. */ @@ -34564,6 +34573,9 @@ export namespace binlogdata { /** VStreamRowsResponse heartbeat. */ public heartbeat: boolean; + /** VStreamRowsResponse throttled_reason. */ + public throttled_reason: string; + /** * Creates a new VStreamRowsResponse instance using the specified properties. * @param [properties] Properties to set diff --git a/web/vtadmin/src/proto/vtadmin.js b/web/vtadmin/src/proto/vtadmin.js index 5952dbed5b6..6bb5c38e205 100644 --- a/web/vtadmin/src/proto/vtadmin.js +++ b/web/vtadmin/src/proto/vtadmin.js @@ -79729,6 +79729,7 @@ export const binlogdata = $root.binlogdata = (() => { * @property {string|null} [keyspace] VEvent keyspace * @property {string|null} [shard] VEvent shard * @property {boolean|null} [throttled] VEvent throttled + * @property {string|null} [throttled_reason] VEvent throttled_reason */ /** @@ -79858,6 +79859,14 @@ export const binlogdata = $root.binlogdata = (() => { */ VEvent.prototype.throttled = false; + /** + * VEvent throttled_reason. + * @member {string} throttled_reason + * @memberof binlogdata.VEvent + * @instance + */ + VEvent.prototype.throttled_reason = ""; + /** * Creates a new VEvent instance using the specified properties. * @function create @@ -79910,6 +79919,8 @@ export const binlogdata = $root.binlogdata = (() => { writer.uint32(/* id 23, wireType 2 =*/186).string(message.shard); if (message.throttled != null && Object.hasOwnProperty.call(message, "throttled")) writer.uint32(/* id 24, wireType 0 =*/192).bool(message.throttled); + if (message.throttled_reason != null && Object.hasOwnProperty.call(message, "throttled_reason")) + writer.uint32(/* id 25, wireType 2 =*/202).string(message.throttled_reason); return writer; }; @@ -80000,6 +80011,10 @@ export const binlogdata = $root.binlogdata = (() => { message.throttled = reader.bool(); break; } + case 25: { + message.throttled_reason = reader.string(); + break; + } default: reader.skipType(tag & 7); break; @@ -80111,6 +80126,9 @@ export const binlogdata = $root.binlogdata = (() => { if (message.throttled != null && message.hasOwnProperty("throttled")) if (typeof message.throttled !== "boolean") return "throttled: boolean expected"; + if (message.throttled_reason != null && message.hasOwnProperty("throttled_reason")) + if (!$util.isString(message.throttled_reason)) + return "throttled_reason: string expected"; return null; }; @@ -80273,6 +80291,8 @@ export const binlogdata = $root.binlogdata = (() => { message.shard = String(object.shard); if (object.throttled != null) message.throttled = Boolean(object.throttled); + if (object.throttled_reason != null) + message.throttled_reason = String(object.throttled_reason); return message; }; @@ -80312,6 +80332,7 @@ export const binlogdata = $root.binlogdata = (() => { object.keyspace = ""; object.shard = ""; object.throttled = false; + object.throttled_reason = ""; } if (message.type != null && message.hasOwnProperty("type")) object.type = options.enums === String ? $root.binlogdata.VEventType[message.type] === undefined ? message.type : $root.binlogdata.VEventType[message.type] : message.type; @@ -80347,6 +80368,8 @@ export const binlogdata = $root.binlogdata = (() => { object.shard = message.shard; if (message.throttled != null && message.hasOwnProperty("throttled")) object.throttled = message.throttled; + if (message.throttled_reason != null && message.hasOwnProperty("throttled_reason")) + object.throttled_reason = message.throttled_reason; return object; }; @@ -81847,6 +81870,7 @@ export const binlogdata = $root.binlogdata = (() => { * @property {query.IRow|null} [lastpk] VStreamRowsResponse lastpk * @property {boolean|null} [throttled] VStreamRowsResponse throttled * @property {boolean|null} [heartbeat] VStreamRowsResponse heartbeat + * @property {string|null} [throttled_reason] VStreamRowsResponse throttled_reason */ /** @@ -81923,6 +81947,14 @@ export const binlogdata = $root.binlogdata = (() => { */ VStreamRowsResponse.prototype.heartbeat = false; + /** + * VStreamRowsResponse throttled_reason. + * @member {string} throttled_reason + * @memberof binlogdata.VStreamRowsResponse + * @instance + */ + VStreamRowsResponse.prototype.throttled_reason = ""; + /** * Creates a new VStreamRowsResponse instance using the specified properties. * @function create @@ -81964,6 +81996,8 @@ export const binlogdata = $root.binlogdata = (() => { writer.uint32(/* id 6, wireType 0 =*/48).bool(message.throttled); if (message.heartbeat != null && Object.hasOwnProperty.call(message, "heartbeat")) writer.uint32(/* id 7, wireType 0 =*/56).bool(message.heartbeat); + if (message.throttled_reason != null && Object.hasOwnProperty.call(message, "throttled_reason")) + writer.uint32(/* id 8, wireType 2 =*/66).string(message.throttled_reason); return writer; }; @@ -82032,6 +82066,10 @@ export const binlogdata = $root.binlogdata = (() => { message.heartbeat = reader.bool(); break; } + case 8: { + message.throttled_reason = reader.string(); + break; + } default: reader.skipType(tag & 7); break; @@ -82108,6 +82146,9 @@ export const binlogdata = $root.binlogdata = (() => { if (message.heartbeat != null && message.hasOwnProperty("heartbeat")) if (typeof message.heartbeat !== "boolean") return "heartbeat: boolean expected"; + if (message.throttled_reason != null && message.hasOwnProperty("throttled_reason")) + if (!$util.isString(message.throttled_reason)) + return "throttled_reason: string expected"; return null; }; @@ -82164,6 +82205,8 @@ export const binlogdata = $root.binlogdata = (() => { message.throttled = Boolean(object.throttled); if (object.heartbeat != null) message.heartbeat = Boolean(object.heartbeat); + if (object.throttled_reason != null) + message.throttled_reason = String(object.throttled_reason); return message; }; @@ -82190,6 +82233,7 @@ export const binlogdata = $root.binlogdata = (() => { object.lastpk = null; object.throttled = false; object.heartbeat = false; + object.throttled_reason = ""; } if (message.fields && message.fields.length) { object.fields = []; @@ -82214,6 +82258,8 @@ export const binlogdata = $root.binlogdata = (() => { object.throttled = message.throttled; if (message.heartbeat != null && message.hasOwnProperty("heartbeat")) object.heartbeat = message.heartbeat; + if (message.throttled_reason != null && message.hasOwnProperty("throttled_reason")) + object.throttled_reason = message.throttled_reason; return object; }; From af99924ebd1dbd0521d6011897347b958514bf72 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:11:16 +0300 Subject: [PATCH 30/36] ThrottlerClient returns CheckResult. Used in vreplication to extract the Summary(), populate ThrottledReason added in proto, then in turn populate new _vt.vreplication.reason_throttled Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/binlog/binlogplayer/binlog_player.go | 4 +- .../binlog/binlogplayer/binlog_player_test.go | 35 ++++++++++ .../schema/vreplication/vreplication.sql | 1 + .../tabletmanager/vreplication/vcopier.go | 6 +- .../tabletmanager/vreplication/vplayer.go | 6 +- .../tabletmanager/vreplication/vreplicator.go | 4 +- go/vt/vttablet/tabletserver/gc/tablegc.go | 2 +- .../tabletserver/throttle/check_result.go | 2 + .../throttle/check_result_test.go | 66 +++++++++++++++++++ .../vttablet/tabletserver/throttle/client.go | 41 +++++++----- .../tabletserver/throttle/throttler_test.go | 33 ++++++---- .../tabletserver/vstreamer/resultstreamer.go | 2 +- .../tabletserver/vstreamer/rowstreamer.go | 4 +- .../tabletserver/vstreamer/vstreamer.go | 17 ++--- 14 files changed, 172 insertions(+), 51 deletions(-) create mode 100644 go/vt/vttablet/tabletserver/throttle/check_result_test.go diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index 05685a54d3e..7936a0760c9 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -669,11 +669,11 @@ func GenerateUpdateHeartbeat(uid int32, timeUpdated int64) (string, error) { } // GenerateUpdateTimeThrottled returns a statement to record the latest throttle time in the _vt.vreplication table. -func GenerateUpdateTimeThrottled(uid int32, timeThrottledUnix int64, componentThrottled string) (string, error) { +func GenerateUpdateTimeThrottled(uid int32, timeThrottledUnix int64, componentThrottled string, reasonThrottled string) (string, error) { if timeThrottledUnix == 0 { return "", fmt.Errorf("timeUpdated cannot be zero") } - return fmt.Sprintf("update _vt.vreplication set time_updated=%v, time_throttled=%v, component_throttled='%v' where id=%v", timeThrottledUnix, timeThrottledUnix, componentThrottled, uid), nil + return fmt.Sprintf("update _vt.vreplication set time_updated=%v, time_throttled=%v, component_throttled='%v', reason_throttled=%v where id=%v", timeThrottledUnix, timeThrottledUnix, componentThrottled, encodeString(MessageTruncate(reasonThrottled)), uid), nil } // StartVReplicationUntil returns a statement to start the replication with a stop position. diff --git a/go/vt/binlog/binlogplayer/binlog_player_test.go b/go/vt/binlog/binlogplayer/binlog_player_test.go index 99b0ef496b3..697733a6d18 100644 --- a/go/vt/binlog/binlogplayer/binlog_player_test.go +++ b/go/vt/binlog/binlogplayer/binlog_player_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/mysql/sqlerror" querypb "vitess.io/vitess/go/vt/proto/query" @@ -454,3 +456,36 @@ func TestReadVReplicationStatus(t *testing.T) { t.Errorf("ReadVReplicationStatus(482821) = %#v, want %#v", got, want) } } + +func TestEncodeString(t *testing.T) { + tcases := []struct { + in, out string + }{ + { + in: "", + out: "''", + }, + { + in: "a", + out: "'a'", + }, + { + in: "here's", + out: "'here\\'s'", + }, + { + in: "online-ddl is denied access due to lag metric value 94.821447 exceeding threshold 5", + out: "'online-ddl is denied access due to lag metric value 94.821447 exceeding threshold 5'", + }, + { + in: "'a','b','c'", + out: "'\\'a\\',\\'b\\',\\'c\\''", + }, + } + for _, tcase := range tcases { + t.Run(tcase.in, func(t *testing.T) { + out := encodeString(tcase.in) + assert.Equal(t, tcase.out, out) + }) + } +} diff --git a/go/vt/sidecardb/schema/vreplication/vreplication.sql b/go/vt/sidecardb/schema/vreplication/vreplication.sql index 8d2ec41d1a6..293997056fd 100644 --- a/go/vt/sidecardb/schema/vreplication/vreplication.sql +++ b/go/vt/sidecardb/schema/vreplication/vreplication.sql @@ -36,6 +36,7 @@ CREATE TABLE IF NOT EXISTS vreplication `workflow_type` int NOT NULL DEFAULT '0', `time_throttled` bigint NOT NULL DEFAULT '0', `component_throttled` varchar(255) NOT NULL DEFAULT '', + `reason_throttled` varchar(1000) NOT NULL DEFAULT '', `workflow_sub_type` int NOT NULL DEFAULT '0', `defer_secondary_keys` tinyint(1) NOT NULL DEFAULT '0', /* diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go index 9057a55707f..47e3798acd0 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go @@ -456,7 +456,7 @@ func (vc *vcopier) copyTable(ctx context.Context, tableName string, copyState ma default: } if rows.Throttled { - _ = vc.vr.updateTimeThrottled(throttlerapp.RowStreamerName) + _ = vc.vr.updateTimeThrottled(throttlerapp.RowStreamerName, rows.ThrottledReason) return nil } if rows.Heartbeat { @@ -464,10 +464,10 @@ func (vc *vcopier) copyTable(ctx context.Context, tableName string, copyState ma return nil } // verify throttler is happy, otherwise keep looping - if vc.vr.vre.throttlerClient.ThrottleCheckOKOrWaitAppName(ctx, throttlerapp.Name(vc.throttlerAppName)) { + if checkResult, ok := vc.vr.vre.throttlerClient.ThrottleCheckOKOrWaitAppName(ctx, throttlerapp.Name(vc.throttlerAppName)); ok { break // out of 'for' loop } else { // we're throttled - _ = vc.vr.updateTimeThrottled(throttlerapp.VCopierName) + _ = vc.vr.updateTimeThrottled(throttlerapp.VCopierName, checkResult.Summary()) } } if !copyWorkQueue.isOpen { diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index c2eba565524..31e26c30e88 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -487,8 +487,8 @@ func (vp *vplayer) applyEvents(ctx context.Context, relay *relayLog) error { return ctx.Err() } // Check throttler. - if !vp.vr.vre.throttlerClient.ThrottleCheckOKOrWaitAppName(ctx, throttlerapp.Name(vp.throttlerAppName)) { - _ = vp.vr.updateTimeThrottled(throttlerapp.VPlayerName) + if checkResult, ok := vp.vr.vre.throttlerClient.ThrottleCheckOKOrWaitAppName(ctx, throttlerapp.Name(vp.throttlerAppName)); !ok { + _ = vp.vr.updateTimeThrottled(throttlerapp.VPlayerName, checkResult.Summary()) continue } @@ -794,7 +794,7 @@ func (vp *vplayer) applyEvent(ctx context.Context, event *binlogdatapb.VEvent, m return io.EOF case binlogdatapb.VEventType_HEARTBEAT: if event.Throttled { - if err := vp.vr.updateTimeThrottled(throttlerapp.VStreamerName); err != nil { + if err := vp.vr.updateTimeThrottled(throttlerapp.VStreamerName, event.ThrottledReason); err != nil { return err } } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index 8a01cf7c8ed..4dcc0e584f6 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -582,13 +582,13 @@ func (vr *vreplicator) throttlerAppName() string { // tablet throttler over time. It also increments the global throttled count to keep // track of how many times in total vreplication has been throttled across all workflows // (both ones that currently exist and ones that no longer do). -func (vr *vreplicator) updateTimeThrottled(appThrottled throttlerapp.Name) error { +func (vr *vreplicator) updateTimeThrottled(appThrottled throttlerapp.Name, reasonThrottled string) error { appName := appThrottled.String() vr.stats.ThrottledCounts.Add([]string{"tablet", appName}, 1) globalStats.ThrottledCount.Add(1) err := vr.throttleUpdatesRateLimiter.Do(func() error { tm := time.Now().Unix() - update, err := binlogplayer.GenerateUpdateTimeThrottled(vr.id, tm, appName) + update, err := binlogplayer.GenerateUpdateTimeThrottled(vr.id, tm, appName, reasonThrottled) if err != nil { return err } diff --git a/go/vt/vttablet/tabletserver/gc/tablegc.go b/go/vt/vttablet/tabletserver/gc/tablegc.go index f1d64aebea3..4d1714532a3 100644 --- a/go/vt/vttablet/tabletserver/gc/tablegc.go +++ b/go/vt/vttablet/tabletserver/gc/tablegc.go @@ -551,7 +551,7 @@ func (collector *TableGC) purge(ctx context.Context) (tableName string, err erro // cancelled return tableName, err } - if !collector.throttlerClient.ThrottleCheckOKOrWait(ctx) { + if _, ok := collector.throttlerClient.ThrottleCheckOKOrWait(ctx); !ok { continue } // OK, we're clear to go! diff --git a/go/vt/vttablet/tabletserver/throttle/check_result.go b/go/vt/vttablet/tabletserver/throttle/check_result.go index e51f1814d8a..105dbe17218 100644 --- a/go/vt/vttablet/tabletserver/throttle/check_result.go +++ b/go/vt/vttablet/tabletserver/throttle/check_result.go @@ -104,6 +104,8 @@ func (c *CheckResult) Summary() string { return fmt.Sprintf("%v is denied access due to %s/%s metric value %v exceeding threshold %v", c.AppName, c.Scope, c.MetricName, c.Value, c.Threshold) case http.StatusNotFound: return fmt.Sprintf("%v is denied access due to unknown or uncollected metric", c.AppName) + case 0: + return "" default: return fmt.Sprintf("unknown status code: %v", c.StatusCode) } diff --git a/go/vt/vttablet/tabletserver/throttle/check_result_test.go b/go/vt/vttablet/tabletserver/throttle/check_result_test.go new file mode 100644 index 00000000000..f5c984c5943 --- /dev/null +++ b/go/vt/vttablet/tabletserver/throttle/check_result_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package throttle + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCheckResultSummary(t *testing.T) { + tcases := []struct { + checkResult *CheckResult + summary string + }{ + { + checkResult: &CheckResult{}, + summary: "", + }, + { + checkResult: &CheckResult{ + StatusCode: http.StatusOK, + AppName: "test", + }, + summary: "test is granted access", + }, + { + checkResult: &CheckResult{ + StatusCode: http.StatusTooManyRequests, + AppName: "test", + MetricName: "bugginess", + Threshold: 100, + Value: 200, + Scope: "self", + }, + summary: "test is denied access due to self/bugginess metric value 200 exceeding threshold 100", + }, + { + checkResult: &CheckResult{ + StatusCode: http.StatusExpectationFailed, + AppName: "test", + }, + summary: "test is explicitly denied access", + }, + } + for _, tcase := range tcases { + t.Run(tcase.summary, func(t *testing.T) { + assert.Equal(t, tcase.summary, tcase.checkResult.Summary()) + }) + } +} diff --git a/go/vt/vttablet/tabletserver/throttle/client.go b/go/vt/vttablet/tabletserver/throttle/client.go index e8eed627e04..972e63724f9 100644 --- a/go/vt/vttablet/tabletserver/throttle/client.go +++ b/go/vt/vttablet/tabletserver/throttle/client.go @@ -31,8 +31,11 @@ const ( throttleCheckDuration = 250 * time.Millisecond ) -var throttleTicks int64 -var throttleInit sync.Once +var ( + throttleTicks int64 + throttleInit sync.Once + emptyCheckResult = &CheckResult{} +) func initThrottleTicker() { throttleInit.Do(func() { @@ -87,14 +90,14 @@ func (c *Client) clearSuccessfulResultsCache() { // The function caches results for a brief amount of time, hence it's safe and efficient to // be called very frequently. // The function is not thread safe. -func (c *Client) ThrottleCheckOK(ctx context.Context, overrideAppName throttlerapp.Name) (throttleCheckOK bool) { +func (c *Client) ThrottleCheckOK(ctx context.Context, overrideAppName throttlerapp.Name) (checkResult *CheckResult, throttleCheckOK bool) { if c == nil { // no client - return true + return emptyCheckResult, true } if c.throttler == nil { // no throttler - return true + return emptyCheckResult, true } checkApp := c.appName if overrideAppName != "" { @@ -104,20 +107,20 @@ func (c *Client) ThrottleCheckOK(ctx context.Context, overrideAppName throttlera defer c.lastSuccessfulThrottleMu.Unlock() if c.lastSuccessfulThrottle[checkApp.String()] >= atomic.LoadInt64(&throttleTicks) { // if last check was OK just very recently there is no need to check again - return true + return emptyCheckResult, true } // It's time to run a throttler check - checkResult := c.throttler.Check(ctx, checkApp.String(), nil, &c.flags) + checkResult = c.throttler.Check(ctx, checkApp.String(), nil, &c.flags) if checkResult.StatusCode != http.StatusOK { - return false + return checkResult, false } for _, metricResult := range checkResult.Metrics { if metricResult.StatusCode != http.StatusOK { - return false + return checkResult, false } } c.lastSuccessfulThrottle[checkApp.String()] = atomic.LoadInt64(&throttleTicks) - return true + return checkResult, true } @@ -125,22 +128,23 @@ func (c *Client) ThrottleCheckOK(ctx context.Context, overrideAppName throttlera // otherwise it briefly sleeps and returns 'false'. // Non-empty appName overrides the default appName. // The function is not thread safe. -func (c *Client) ThrottleCheckOKOrWaitAppName(ctx context.Context, appName throttlerapp.Name) bool { - if c.ThrottleCheckOK(ctx, appName) { - return true +func (c *Client) ThrottleCheckOKOrWaitAppName(ctx context.Context, appName throttlerapp.Name) (checkResult *CheckResult, throttleCheckOK bool) { + checkResult, throttleCheckOK = c.ThrottleCheckOK(ctx, appName) + if throttleCheckOK { + return checkResult, true } if ctx.Err() != nil { // context expired, skip sleeping - return false + return checkResult, false } time.Sleep(throttleCheckDuration) - return false + return checkResult, false } // ThrottleCheckOKOrWait checks the throttler; if throttler is satisfied, the function returns 'true' immediately, // otherwise it briefly sleeps and returns 'false'. // The function is not thread safe. -func (c *Client) ThrottleCheckOKOrWait(ctx context.Context) bool { +func (c *Client) ThrottleCheckOKOrWait(ctx context.Context) (checkResult *CheckResult, throttleCheckOK bool) { return c.ThrottleCheckOKOrWaitAppName(ctx, "") } @@ -148,7 +152,10 @@ func (c *Client) ThrottleCheckOKOrWait(ctx context.Context) bool { // The function sleeps between throttle checks. // The function is not thread safe. func (c *Client) Throttle(ctx context.Context) { - for !c.ThrottleCheckOKOrWait(ctx) { + for { + if _, ok := c.ThrottleCheckOKOrWait(ctx); ok { + return + } // The function incorporates a bit of sleep so this is not a busy wait. } } diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index 1ed0ce3f126..74b3d91b53c 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -1316,8 +1316,10 @@ func TestProbesWhileOperating(t *testing.T) { throttler.refreshInventory(ctx) }) { - checkOK := client.ThrottleCheckOK(ctx, "") + checkResult, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) // we expect threshold exceeded + assert.NotNil(t, checkResult) + assert.Contains(t, checkResult.Summary(), "test is denied access due to") } }) @@ -1328,7 +1330,7 @@ func TestProbesWhileOperating(t *testing.T) { throttler.refreshInventory(ctx) }) { - checkOK := client.ThrottleCheckOK(ctx, "") + _, checkOK := client.ThrottleCheckOK(ctx, "") assert.True(t, checkOK) } }) @@ -1339,8 +1341,10 @@ func TestProbesWhileOperating(t *testing.T) { }) client.clearSuccessfulResultsCache() // ensure we don't read the successful result from the test above { - checkOK := client.ThrottleCheckOK(ctx, "") + checkResult, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) + assert.NotNil(t, checkResult) + assert.Contains(t, checkResult.Summary(), "test is denied access due to") } }) }) @@ -1405,8 +1409,9 @@ func TestProbesWhileOperating(t *testing.T) { throttler.refreshInventory(ctx) }) { - checkOK := client.ThrottleCheckOK(ctx, "") + checkResult, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) // we expect threshold exceeded + assert.NotNil(t, checkResult) } }) @@ -1417,7 +1422,7 @@ func TestProbesWhileOperating(t *testing.T) { throttler.refreshInventory(ctx) }) { - checkOK := client.ThrottleCheckOK(ctx, "") + _, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) // 0.95 still too low for custom query } }) @@ -1427,7 +1432,7 @@ func TestProbesWhileOperating(t *testing.T) { throttler.refreshInventory(ctx) }) { - checkOK := client.ThrottleCheckOK(ctx, "") + _, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) // 15 still too low for custom query because primary has 17 } }) @@ -1437,7 +1442,7 @@ func TestProbesWhileOperating(t *testing.T) { throttler.refreshInventory(ctx) }) { - checkOK := client.ThrottleCheckOK(ctx, "") + _, checkOK := client.ThrottleCheckOK(ctx, "") assert.True(t, checkOK) } }) @@ -1448,7 +1453,7 @@ func TestProbesWhileOperating(t *testing.T) { }) client.clearSuccessfulResultsCache() // ensure we don't read the successful result from the test above { - checkOK := client.ThrottleCheckOK(ctx, "") + _, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) } }) @@ -2055,7 +2060,7 @@ func TestReplica(t *testing.T) { }) t.Run("client, OK", func(t *testing.T) { client := NewBackgroundClient(throttler, throttlerapp.TestingName, base.UndefinedScope) - checkOK := client.ThrottleCheckOK(ctx, "") + _, checkOK := client.ThrottleCheckOK(ctx, "") assert.True(t, checkOK) }) t.Run("client, metrics names mapped, OK", func(t *testing.T) { @@ -2063,7 +2068,7 @@ func TestReplica(t *testing.T) { throttler.appCheckedMetrics.Set(throttlerapp.TestingName.String(), base.MetricNames{base.LagMetricName, base.ThreadsRunningMetricName}, cache.DefaultExpiration) defer throttler.appCheckedMetrics.Delete(throttlerapp.TestingName.String()) client := NewBackgroundClient(throttler, throttlerapp.TestingName, base.UndefinedScope) - checkOK := client.ThrottleCheckOK(ctx, "") + _, checkOK := client.ThrottleCheckOK(ctx, "") assert.True(t, checkOK) }) t.Run("client, metrics names mapped, not OK", func(t *testing.T) { @@ -2071,8 +2076,10 @@ func TestReplica(t *testing.T) { throttler.appCheckedMetrics.Set(throttlerapp.TestingName.String(), base.MetricNames{base.LagMetricName, base.LoadAvgMetricName, base.ThreadsRunningMetricName}, cache.DefaultExpiration) defer throttler.appCheckedMetrics.Delete(throttlerapp.TestingName.String()) client := NewBackgroundClient(throttler, throttlerapp.TestingName, base.UndefinedScope) - checkOK := client.ThrottleCheckOK(ctx, "") + checkResult, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) + assert.NotNil(t, checkResult) + assert.Contains(t, checkResult.Summary(), "test is denied access due to") }) t.Run("custom query, metrics", func(t *testing.T) { @@ -2106,8 +2113,10 @@ func TestReplica(t *testing.T) { }) t.Run("client, not OK", func(t *testing.T) { client := NewBackgroundClient(throttler, throttlerapp.TestingName, base.SelfScope) - checkOK := client.ThrottleCheckOK(ctx, "") + checkResult, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) + assert.NotNil(t, checkResult) + assert.Contains(t, checkResult.Summary(), "test is denied access due to") }) }() }) diff --git a/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go index 4632bea672b..b6294cd1939 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go @@ -109,7 +109,7 @@ func (rs *resultStreamer) Stream() error { } // check throttler. - if !rs.vse.throttlerClient.ThrottleCheckOKOrWaitAppName(rs.ctx, throttlerapp.ResultStreamerName) { + if _, ok := rs.vse.throttlerClient.ThrottleCheckOKOrWaitAppName(rs.ctx, throttlerapp.ResultStreamerName); !ok { logger.Infof("throttled.") continue } diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go index bb8ff7af85f..6015590dad7 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go @@ -388,9 +388,9 @@ func (rs *rowStreamer) streamQuery(send func(*binlogdatapb.VStreamRowsResponse) } // check throttler. - if !rs.vse.throttlerClient.ThrottleCheckOKOrWaitAppName(rs.ctx, throttlerapp.RowStreamerName) { + if checkResult, ok := rs.vse.throttlerClient.ThrottleCheckOKOrWaitAppName(rs.ctx, throttlerapp.RowStreamerName); !ok { throttleResponseRateLimiter.Do(func() error { - return safeSend(&binlogdatapb.VStreamRowsResponse{Throttled: true}) + return safeSend(&binlogdatapb.VStreamRowsResponse{Throttled: true, ThrottledReason: checkResult.Summary()}) }) logger.Infof("throttled.") continue diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 3413c53d811..634c9a5d40c 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -287,17 +287,18 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog hbTimer := time.NewTimer(HeartbeatTime) defer hbTimer.Stop() - injectHeartbeat := func(throttled bool) error { + injectHeartbeat := func(throttled bool, throttledReason string) error { now := time.Now().UnixNano() select { case <-ctx.Done(): return vterrors.Errorf(vtrpcpb.Code_CANCELED, "context has expired") default: err := bufferAndTransmit(&binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_HEARTBEAT, - Timestamp: now / 1e9, - CurrentTime: now, - Throttled: throttled, + Type: binlogdatapb.VEventType_HEARTBEAT, + Timestamp: now / 1e9, + CurrentTime: now, + Throttled: throttled, + ThrottledReason: throttledReason, }) return err } @@ -309,7 +310,7 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog defer throttledHeartbeatsRateLimiter.Stop() for { // check throttler. - if !vs.vse.throttlerClient.ThrottleCheckOKOrWaitAppName(ctx, vs.throttlerApp) { + if checkResult, ok := vs.vse.throttlerClient.ThrottleCheckOKOrWaitAppName(ctx, vs.throttlerApp); !ok { // make sure to leave if context is cancelled select { case <-ctx.Done(): @@ -318,7 +319,7 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog // do nothing special } throttledHeartbeatsRateLimiter.Do(func() error { - return injectHeartbeat(true) + return injectHeartbeat(true, checkResult.Summary()) }) // we won't process events, until we're no longer throttling logger.Infof("throttled.") @@ -393,7 +394,7 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog case <-ctx.Done(): return nil case <-hbTimer.C: - if err := injectHeartbeat(false); err != nil { + if err := injectHeartbeat(false, ""); err != nil { if err == io.EOF { return nil } From b31a552a3d7db0598d649ef9e5993bd7406640ad Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 18 Jul 2024 16:09:57 +0300 Subject: [PATCH 31/36] copy _vt.vreplication.reason_throttled to _vt.schema_migrations.reason_throttled Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../sidecardb/schema/onlineddl/schema_migrations.sql | 1 + go/vt/vttablet/onlineddl/executor.go | 12 ++++++++++-- go/vt/vttablet/onlineddl/schema.go | 4 +++- go/vt/vttablet/onlineddl/vrepl.go | 1 + 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/go/vt/sidecardb/schema/onlineddl/schema_migrations.sql b/go/vt/sidecardb/schema/onlineddl/schema_migrations.sql index 2926ec76f28..82d0c221f0e 100644 --- a/go/vt/sidecardb/schema/onlineddl/schema_migrations.sql +++ b/go/vt/sidecardb/schema/onlineddl/schema_migrations.sql @@ -63,6 +63,7 @@ CREATE TABLE IF NOT EXISTS schema_migrations `special_plan` text NOT NULL, `last_throttled_timestamp` timestamp NULL DEFAULT NULL, `component_throttled` tinytext NOT NULL, + `reason_throttled` text NOT NULL, `cancelled_timestamp` timestamp NULL DEFAULT NULL, `postpone_launch` tinyint unsigned NOT NULL DEFAULT '0', `stage` text NOT NULL, diff --git a/go/vt/vttablet/onlineddl/executor.go b/go/vt/vttablet/onlineddl/executor.go index 1d2137b7426..fe0f844ab91 100644 --- a/go/vt/vttablet/onlineddl/executor.go +++ b/go/vt/vttablet/onlineddl/executor.go @@ -3640,6 +3640,7 @@ func (e *Executor) readVReplStream(ctx context.Context, uuid string, okIfMissing timeHeartbeat: row.AsInt64("time_heartbeat", 0), timeThrottled: row.AsInt64("time_throttled", 0), componentThrottled: row.AsString("component_throttled", ""), + reasonThrottled: row.AsString("reason_throttled", ""), transactionTimestamp: row.AsInt64("transaction_timestamp", 0), state: binlogdatapb.VReplicationWorkflowState(binlogdatapb.VReplicationWorkflowState_value[row.AsString("state", "")]), message: row.AsString("message", ""), @@ -3872,7 +3873,7 @@ func (e *Executor) reviewRunningMigrations(ctx context.Context) (countRunnning i _ = e.updateMigrationETASecondsByProgress(ctx, uuid) if s.timeThrottled != 0 { // Avoid creating a 0000-00-00 00:00:00 timestamp - _ = e.updateMigrationLastThrottled(ctx, uuid, time.Unix(s.timeThrottled, 0), s.componentThrottled) + _ = e.updateMigrationLastThrottled(ctx, uuid, time.Unix(s.timeThrottled, 0), s.componentThrottled, s.reasonThrottled) } if onlineDDL.StrategySetting().IsInOrderCompletion() { // We will fail an in-order migration if there's _prior_ migrations within the same migration-context @@ -4575,10 +4576,17 @@ func (e *Executor) updateMigrationETASecondsByProgress(ctx context.Context, uuid return err } -func (e *Executor) updateMigrationLastThrottled(ctx context.Context, uuid string, lastThrottledTime time.Time, throttledCompnent string) error { +func (e *Executor) updateMigrationLastThrottled( + ctx context.Context, + uuid string, + lastThrottledTime time.Time, + throttledCompnent string, + reasonThrottled string, +) error { query, err := sqlparser.ParseAndBind(sqlUpdateLastThrottled, sqltypes.StringBindVariable(lastThrottledTime.Format(sqltypes.TimestampFormat)), sqltypes.StringBindVariable(throttledCompnent), + sqltypes.StringBindVariable(reasonThrottled), sqltypes.StringBindVariable(uuid), ) if err != nil { diff --git a/go/vt/vttablet/onlineddl/schema.go b/go/vt/vttablet/onlineddl/schema.go index 30f132bd0e3..d964252c3e2 100644 --- a/go/vt/vttablet/onlineddl/schema.go +++ b/go/vt/vttablet/onlineddl/schema.go @@ -242,7 +242,7 @@ const ( migration_uuid=%a ` sqlUpdateLastThrottled = `UPDATE _vt.schema_migrations - SET last_throttled_timestamp=%a, component_throttled=%a + SET last_throttled_timestamp=%a, component_throttled=%a, reason_throttled=%a WHERE migration_uuid=%a ` @@ -428,6 +428,7 @@ const ( last_throttled_timestamp, cancelled_timestamp, component_throttled, + reason_throttled, postpone_launch, postpone_completion, is_immediate_operation, @@ -581,6 +582,7 @@ const ( time_heartbeat, time_throttled, component_throttled, + reason_throttled, state, message, rows_copied diff --git a/go/vt/vttablet/onlineddl/vrepl.go b/go/vt/vttablet/onlineddl/vrepl.go index cde2f276563..42fe33a855f 100644 --- a/go/vt/vttablet/onlineddl/vrepl.go +++ b/go/vt/vttablet/onlineddl/vrepl.go @@ -59,6 +59,7 @@ type VReplStream struct { timeHeartbeat int64 timeThrottled int64 componentThrottled string + reasonThrottled string transactionTimestamp int64 state binlogdatapb.VReplicationWorkflowState message string From 313675c55049f0731bd5ca77a3da00327420c71e Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 18 Jul 2024 16:13:16 +0300 Subject: [PATCH 32/36] test 'reason_throttled' in endtoend Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go index 83fe6bea988..afd5fb7eba4 100644 --- a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go +++ b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go @@ -397,6 +397,8 @@ func TestSchemaChange(t *testing.T) { assert.GreaterOrEqual(t, lastThrottledTimestamp, startedTimestamp) component := row.AsString("component_throttled", "") assert.Contains(t, []string{throttlerapp.VCopierName.String(), throttlerapp.VPlayerName.String()}, component) + reason := row.AsString("reason_throttled", "") + assert.Contains(t, reason, "online-ddl is denied access due to") // unthrottle onlineddl.UnthrottleAllMigrations(t, &vtParams) From 5a3aea7a6cd81b2cc0f4fb9b54ac23f63f071fe0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 18 Jul 2024 16:18:13 +0300 Subject: [PATCH 33/36] fix expected message Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go index afd5fb7eba4..70efe4ec8a4 100644 --- a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go +++ b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go @@ -398,7 +398,7 @@ func TestSchemaChange(t *testing.T) { component := row.AsString("component_throttled", "") assert.Contains(t, []string{throttlerapp.VCopierName.String(), throttlerapp.VPlayerName.String()}, component) reason := row.AsString("reason_throttled", "") - assert.Contains(t, reason, "online-ddl is denied access due to") + assert.Contains(t, reason, "is explicitly denied access") // unthrottle onlineddl.UnthrottleAllMigrations(t, &vtParams) From 2cd72f1d47112dd06ab49a9d9d471c433c0d3fa6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 28 Jul 2024 08:06:19 +0300 Subject: [PATCH 34/36] string format Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/check_result.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/check_result.go b/go/vt/vttablet/tabletserver/throttle/check_result.go index 105dbe17218..ad32ba33d6f 100644 --- a/go/vt/vttablet/tabletserver/throttle/check_result.go +++ b/go/vt/vttablet/tabletserver/throttle/check_result.go @@ -95,15 +95,15 @@ func (c *CheckResult) IsOK() bool { func (c *CheckResult) Summary() string { switch c.StatusCode { case http.StatusOK: - return fmt.Sprintf("%v is granted access", c.AppName) + return fmt.Sprintf("%s is granted access", c.AppName) case http.StatusExpectationFailed: - return fmt.Sprintf("%v is explicitly denied access", c.AppName) + return fmt.Sprintf("%s is explicitly denied access", c.AppName) case http.StatusInternalServerError: - return fmt.Sprintf("%v is denied access due to unexpected error: %v", c.AppName, c.Error) + return fmt.Sprintf("%s is denied access due to unexpected error: %v", c.AppName, c.Error) case http.StatusTooManyRequests: - return fmt.Sprintf("%v is denied access due to %s/%s metric value %v exceeding threshold %v", c.AppName, c.Scope, c.MetricName, c.Value, c.Threshold) + return fmt.Sprintf("%s is denied access due to %s/%s metric value %v exceeding threshold %v", c.AppName, c.Scope, c.MetricName, c.Value, c.Threshold) case http.StatusNotFound: - return fmt.Sprintf("%v is denied access due to unknown or uncollected metric", c.AppName) + return fmt.Sprintf("%s is denied access due to unknown or uncollected metric", c.AppName) case 0: return "" default: From d6e3f458bd6ca59fdefc1e43927704b9c402e185 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 28 Jul 2024 08:07:45 +0300 Subject: [PATCH 35/36] remove extraneous comment indicator. Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/throttle/throttler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index 74b3d91b53c..22d150524ad 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -906,7 +906,7 @@ func TestIsAppThrottled(t *testing.T) { // continuing previous test, we had 3 throttled apps. "all" is a new app being throttled. assert.Equal(t, 4, throttler.throttledApps.ItemCount()) }) - // // + // t.Run("unthrottle", func(t *testing.T) { throttler.UnthrottleApp("app1") throttler.UnthrottleApp("app2") From 0eef755171139f4955ce100345ace42f0c12d625 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 28 Jul 2024 08:14:01 +0300 Subject: [PATCH 36/36] formalize expected check result summary based on app name Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/throttle/throttler_test.go | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index 22d150524ad..508f542478e 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -418,7 +418,7 @@ func TestApplyThrottlerConfigMetricThresholds(t *testing.T) { assert.EqualValues(t, 0.3, checkResult.Value) // self lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) t.Run("apply low threshold", func(t *testing.T) { assert.Equal(t, 0.75, throttler.GetMetricsThreshold()) @@ -441,7 +441,7 @@ func TestApplyThrottlerConfigMetricThresholds(t *testing.T) { assert.EqualValues(t, 0.3, checkResult.Value, "unexpected result: %+v", checkResult) // self lag value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "test is denied access due to self/lag metric value") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to self/lag metric value") }) t.Run("apply low threshold but high 'lag' override", func(t *testing.T) { throttlerConfig := &topodatapb.ThrottlerConfig{ @@ -465,7 +465,7 @@ func TestApplyThrottlerConfigMetricThresholds(t *testing.T) { assert.EqualValues(t, 0.3, checkResult.Value, "unexpected result: %+v", checkResult) // self lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) }) @@ -523,7 +523,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "test is denied access due to shard/lag metric value") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to shard/lag metric value") }) t.Run("apply high lag threshold", func(t *testing.T) { throttlerConfig.Threshold = 4444.0 @@ -538,7 +538,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // self lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) }) t.Run("apply low 'loadavg' threshold", func(t *testing.T) { @@ -553,7 +553,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) }) t.Run("assign 'loadavg' to test app", func(t *testing.T) { @@ -572,7 +572,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // self loadavg value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "test is denied access due to self/loadavg metric value") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to self/loadavg metric value") }) }) t.Run("assign 'shard/loadavg' to test app", func(t *testing.T) { @@ -591,7 +591,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 5.1, checkResult.Value) // shard loadavg value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "test is denied access due to shard/loadavg metric value") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to shard/loadavg metric value") }) }) t.Run("assign 'lag,loadavg' to test app", func(t *testing.T) { @@ -609,7 +609,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // self loadavg value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) - assert.Contains(t, checkResult.Summary(), "test is denied access due to self/loadavg metric value") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to self/loadavg metric value") }) }) t.Run("assign 'lag,shard/loadavg' to test app", func(t *testing.T) { @@ -627,7 +627,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 5.1, checkResult.Value) // shard loadavg value assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) - assert.Contains(t, checkResult.Summary(), "test is denied access due to shard/loadavg metric value") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to shard/loadavg metric value") }) }) t.Run("clear 'loadavg' threshold", func(t *testing.T) { @@ -641,7 +641,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 1, len(checkResult.Metrics), "unexpected metrics: %+v", checkResult.Metrics) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) }) t.Run("assign 'lag,threads_running' to test app", func(t *testing.T) { @@ -659,7 +659,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) }) t.Run("assign 'custom,loadavg' to 'all' app", func(t *testing.T) { @@ -677,7 +677,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // loadavg self value exceeds threshold assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) - assert.Contains(t, checkResult.Summary(), "all is denied access due to self/loadavg metric value") + assert.Contains(t, checkResult.Summary(), throttlerapp.AllName.String()+" is denied access due to self/loadavg metric value") }) t.Run("check 'test' after assignment", func(t *testing.T) { // "test" app unaffected by 'all' assignment, because it has @@ -692,7 +692,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) t.Run("'online-ddl' app affected by 'all'", func(t *testing.T) { // "online-ddl" app is affected by 'all' assignment, because it has @@ -706,7 +706,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // loadavg self value exceeds threshold assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) - assert.Contains(t, checkResult.Summary(), "all is denied access due to self/loadavg metric value") + assert.Contains(t, checkResult.Summary(), throttlerapp.AllName.String()+" is denied access due to self/loadavg metric value") }) }) t.Run("'vreplication:online-ddl:12345' app affected by 'all'", func(t *testing.T) { @@ -717,7 +717,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 2.718, checkResult.Value) // loadavg self value exceeds threshold assert.NotEqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) - assert.Contains(t, checkResult.Summary(), "all is denied access due to self/loadavg metric value") + assert.Contains(t, checkResult.Summary(), throttlerapp.AllName.String()+" is denied access due to self/loadavg metric value") }) t.Run("'vreplication:online-ddl:test' app affected by 'test' and not by 'all'", func(t *testing.T) { // "vreplication:online-ddl:test" app is affected by 'test' assignment, because it has @@ -727,7 +727,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) t.Run("deassign metrics from 'all' app", func(t *testing.T) { delete(throttlerConfig.AppCheckedMetrics, throttlerapp.AllName.String()) @@ -742,7 +742,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "all is granted access") + assert.Contains(t, checkResult.Summary(), throttlerapp.AllName.String()+" is granted access") }) t.Run("check 'test' after assignment", func(t *testing.T) { // "test" app unaffected by the entire 'all' assignment, because it has @@ -757,7 +757,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Equal(t, 2, len(checkResult.Metrics)) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) t.Run("'online-ddl' no longer has 'all' impact", func(t *testing.T) { // "online-ddl" app is affected by 'all' assignment, because it has @@ -771,7 +771,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode, "unexpected result: %+v", checkResult) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "online-ddl is granted access") + assert.Contains(t, checkResult.Summary(), throttlerapp.OnlineDDLName.String()+" is granted access") }) }) @@ -788,7 +788,7 @@ func TestApplyThrottlerConfigAppCheckedMetrics(t *testing.T) { assert.EqualValues(t, 0.9, checkResult.Value) // shard lag value assert.EqualValues(t, http.StatusOK, checkResult.StatusCode) assert.Len(t, checkResult.Metrics, 1) - assert.Contains(t, checkResult.Summary(), "test is granted access") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is granted access") }) }) @@ -1319,7 +1319,7 @@ func TestProbesWhileOperating(t *testing.T) { checkResult, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) // we expect threshold exceeded assert.NotNil(t, checkResult) - assert.Contains(t, checkResult.Summary(), "test is denied access due to") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to") } }) @@ -1344,7 +1344,7 @@ func TestProbesWhileOperating(t *testing.T) { checkResult, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) assert.NotNil(t, checkResult) - assert.Contains(t, checkResult.Summary(), "test is denied access due to") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to") } }) }) @@ -2079,7 +2079,7 @@ func TestReplica(t *testing.T) { checkResult, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) assert.NotNil(t, checkResult) - assert.Contains(t, checkResult.Summary(), "test is denied access due to") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to") }) t.Run("custom query, metrics", func(t *testing.T) { @@ -2116,7 +2116,7 @@ func TestReplica(t *testing.T) { checkResult, checkOK := client.ThrottleCheckOK(ctx, "") assert.False(t, checkOK) assert.NotNil(t, checkResult) - assert.Contains(t, checkResult.Summary(), "test is denied access due to") + assert.Contains(t, checkResult.Summary(), testAppName.String()+" is denied access due to") }) }() })