From de33a39a9db727aee46e913afba6066d87377895 Mon Sep 17 00:00:00 2001 From: Noble Mittal <62551163+beingnoble03@users.noreply.github.com> Date: Tue, 28 Jan 2025 02:43:47 +0530 Subject: [PATCH 01/68] test: Add unit tests for `vtctl/workflow` (#17618) Signed-off-by: Noble Mittal --- go/vt/vtctl/workflow/framework_test.go | 31 +++-- go/vt/vtctl/workflow/server_test.go | 82 ++++++++++++ go/vt/vtctl/workflow/traffic_switcher_test.go | 121 ++++++++++++++++++ go/vt/vtctl/workflow/utils_test.go | 76 +++++++++++ 4 files changed, 302 insertions(+), 8 deletions(-) diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go index a2c1b2ef8e3..fad48e31e0c 100644 --- a/go/vt/vtctl/workflow/framework_test.go +++ b/go/vt/vtctl/workflow/framework_test.go @@ -272,7 +272,7 @@ type testTMClient struct { createVReplicationWorkflowRequests map[uint32]*createVReplicationWorkflowRequestResponse readVReplicationWorkflowRequests map[uint32]*readVReplicationWorkflowRequestResponse updateVReplicationWorklowsRequests map[uint32]*tabletmanagerdatapb.UpdateVReplicationWorkflowsRequest - applySchemaRequests map[uint32]*applySchemaRequestResponse + applySchemaRequests map[uint32][]*applySchemaRequestResponse primaryPositions map[uint32]string vdiffRequests map[uint32]*vdiffRequestResponse refreshStateErrors map[uint32]error @@ -296,7 +296,7 @@ func newTestTMClient(env *testEnv) *testTMClient { createVReplicationWorkflowRequests: make(map[uint32]*createVReplicationWorkflowRequestResponse), readVReplicationWorkflowRequests: make(map[uint32]*readVReplicationWorkflowRequestResponse), updateVReplicationWorklowsRequests: make(map[uint32]*tabletmanagerdatapb.UpdateVReplicationWorkflowsRequest), - applySchemaRequests: make(map[uint32]*applySchemaRequestResponse), + applySchemaRequests: make(map[uint32][]*applySchemaRequestResponse), readVReplicationWorkflowsResponses: make(map[string][]*tabletmanagerdatapb.ReadVReplicationWorkflowsResponse), primaryPositions: make(map[uint32]string), vdiffRequests: make(map[uint32]*vdiffRequestResponse), @@ -398,10 +398,9 @@ func (tmc *testTMClient) GetSchema(ctx context.Context, tablet *topodatapb.Table schemaDefn := &tabletmanagerdatapb.SchemaDefinition{} for _, table := range req.Tables { if table == "/.*/" { - // Special case of all tables in keyspace. - for key, tableDefn := range tmc.schema { + for key, schemaDefinition := range tmc.schema { if strings.HasPrefix(key, tablet.Keyspace+".") { - schemaDefn.TableDefinitions = append(schemaDefn.TableDefinitions, tableDefn.TableDefinitions...) + schemaDefn.TableDefinitions = append(schemaDefn.TableDefinitions, schemaDefinition.TableDefinitions...) } } break @@ -414,6 +413,12 @@ func (tmc *testTMClient) GetSchema(ctx context.Context, tablet *topodatapb.Table } schemaDefn.TableDefinitions = append(schemaDefn.TableDefinitions, tableDefn.TableDefinitions...) } + for key, schemaDefinition := range tmc.schema { + if strings.HasPrefix(key, tablet.Keyspace) { + schemaDefn.DatabaseSchema = schemaDefinition.DatabaseSchema + break + } + } return schemaDefn, nil } @@ -508,10 +513,10 @@ func (tmc *testTMClient) expectApplySchemaRequest(tabletID uint32, req *applySch defer tmc.mu.Unlock() if tmc.applySchemaRequests == nil { - tmc.applySchemaRequests = make(map[uint32]*applySchemaRequestResponse) + tmc.applySchemaRequests = make(map[uint32][]*applySchemaRequestResponse) } - tmc.applySchemaRequests[tabletID] = req + tmc.applySchemaRequests[tabletID] = append(tmc.applySchemaRequests[tabletID], req) } // Note: ONLY breaks up change.SQL into individual statements and executes it. Does NOT fully implement ApplySchema. @@ -519,11 +524,17 @@ func (tmc *testTMClient) ApplySchema(ctx context.Context, tablet *topodatapb.Tab tmc.mu.Lock() defer tmc.mu.Unlock() - if expect, ok := tmc.applySchemaRequests[tablet.Alias.Uid]; ok { + if requests, ok := tmc.applySchemaRequests[tablet.Alias.Uid]; ok { + if len(requests) == 0 { + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected ApplySchema request on tablet %s: got %+v", + topoproto.TabletAliasString(tablet.Alias), change) + } + expect := requests[0] if !reflect.DeepEqual(change, expect.change) { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected ApplySchema request on tablet %s: got %+v, want %+v", topoproto.TabletAliasString(tablet.Alias), change, expect.change) } + tmc.applySchemaRequests[tablet.Alias.Uid] = tmc.applySchemaRequests[tablet.Alias.Uid][1:] return expect.res, expect.err } @@ -779,6 +790,10 @@ func (tmc *testTMClient) getVReplicationWorkflowsResponse(key string) *tabletman return resp } +func (tmc *testTMClient) ReloadSchema(ctx context.Context, tablet *topodatapb.Tablet, waitPosition string) error { + return nil +} + // // Utility / helper functions. // diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index ae34dabfc19..4676b732245 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -34,10 +34,12 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/mysqlctl/tmutils" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vtenv" + "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" "vitess.io/vitess/go/vt/vttablet/tmclient" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" @@ -2315,3 +2317,83 @@ func TestWorkflowStatus(t *testing.T) { assert.Equal(t, float32(50), stateTable1.RowsPercentage) assert.Equal(t, float32(50), stateTable2.RowsPercentage) } + +func TestDeleteShard(t *testing.T) { + ctx := context.Background() + + sourceKeyspace := &testKeyspace{"source_keyspace", []string{"-"}} + targetKeyspace := &testKeyspace{"target_keyspace", []string{"-"}} + + te := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer te.close() + + // Verify that shard exists. + si, err := te.ts.GetShard(ctx, targetKeyspace.KeyspaceName, targetKeyspace.ShardNames[0]) + require.NoError(t, err) + require.NotNil(t, si) + + // Expect to fail if recursive is false. + err = te.ws.DeleteShard(ctx, targetKeyspace.KeyspaceName, targetKeyspace.ShardNames[0], false, true) + assert.ErrorContains(t, err, "shard target_keyspace/- still has 1 tablets in cell") + + // Should not throw error if given keyspace or shard is invalid. + err = te.ws.DeleteShard(ctx, "invalid_keyspace", "-", false, true) + assert.NoError(t, err) + + // Successful shard delete. + err = te.ws.DeleteShard(ctx, targetKeyspace.KeyspaceName, targetKeyspace.ShardNames[0], true, true) + assert.NoError(t, err) + + // Check if the shard was deleted. + _, err = te.ts.GetShard(ctx, targetKeyspace.KeyspaceName, targetKeyspace.ShardNames[0]) + assert.ErrorContains(t, err, "node doesn't exist") +} + +func TestCopySchemaShard(t *testing.T) { + ctx := context.Background() + + sourceKeyspace := &testKeyspace{"source_keyspace", []string{"-"}} + targetKeyspace := &testKeyspace{"target_keyspace", []string{"-"}} + + te := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer te.close() + + sqlSchema := `create table t1(id bigint(20) unsigned auto_increment, msg varchar(64), primary key (id)) Engine=InnoDB;` + te.tmc.schema[fmt.Sprintf("%s.t1", sourceKeyspace.KeyspaceName)] = &tabletmanagerdatapb.SchemaDefinition{ + DatabaseSchema: "CREATE DATABASE {{.DatabaseName}}", + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: "t1", + Schema: sqlSchema, + Columns: []string{ + "id", + "msg", + }, + Type: tmutils.TableBaseTable, + }, + }, + } + + // Expect queries on target shards + te.tmc.expectApplySchemaRequest(200, &applySchemaRequestResponse{ + change: &tmutils.SchemaChange{ + SQL: "CREATE DATABASE `vt_target_keyspace`", + Force: false, + AllowReplication: true, + SQLMode: vreplication.SQLMode, + }, + }) + te.tmc.expectApplySchemaRequest(200, &applySchemaRequestResponse{ + change: &tmutils.SchemaChange{ + SQL: sqlSchema, + Force: false, + AllowReplication: true, + SQLMode: vreplication.SQLMode, + }, + }) + + sourceTablet := te.tablets[sourceKeyspace.KeyspaceName][100] + err := te.ws.CopySchemaShard(ctx, sourceTablet.Alias, []string{"/.*/"}, nil, false, targetKeyspace.KeyspaceName, "-", 1*time.Second, true) + assert.NoError(t, err) + assert.Empty(t, te.tmc.applySchemaRequests[200]) +} diff --git a/go/vt/vtctl/workflow/traffic_switcher_test.go b/go/vt/vtctl/workflow/traffic_switcher_test.go index 2cf998eb8e4..a7da91174b9 100644 --- a/go/vt/vtctl/workflow/traffic_switcher_test.go +++ b/go/vt/vtctl/workflow/traffic_switcher_test.go @@ -30,6 +30,7 @@ import ( "vitess.io/vitess/go/sqlescape" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/mysqlctl/tmutils" + "vitess.io/vitess/go/vt/proto/binlogdata" "vitess.io/vitess/go/vt/proto/vschema" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/topo" @@ -912,3 +913,123 @@ func TestAddParticipatingTablesToKeyspace(t *testing.T) { assert.Len(t, vs.Tables["t1"].ColumnVindexes, 2) assert.Len(t, vs.Tables["t2"].ColumnVindexes, 1) } + +func TestCancelMigration_TABLES(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + workflowName := "wf1" + tableName := "t1" + + sourceKeyspace := &testKeyspace{ + KeyspaceName: "sourceks", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "targetks", + ShardNames: []string{"0"}, + } + + schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ + tableName: { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: tableName, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + }, + }, + }, + } + + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + env.tmc.schema = schema + + ts, _, err := env.ws.getWorkflowState(ctx, targetKeyspace.KeyspaceName, workflowName) + require.NoError(t, err) + + sm, err := BuildStreamMigrator(ctx, ts, false, sqlparser.NewTestParser()) + require.NoError(t, err) + + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running', message='' where db_name='vt_targetks' and workflow='wf1'", &sqltypes.Result{}) + env.tmc.expectVRQuery(100, "delete from _vt.vreplication where db_name = 'vt_sourceks' and workflow = 'wf1_reverse'", &sqltypes.Result{}) + + ctx, _, err = env.ts.LockKeyspace(ctx, targetKeyspace.KeyspaceName, "test") + require.NoError(t, err) + + ctx, _, err = env.ts.LockKeyspace(ctx, sourceKeyspace.KeyspaceName, "test") + require.NoError(t, err) + + err = topo.CheckKeyspaceLocked(ctx, ts.targetKeyspace) + require.NoError(t, err) + + err = topo.CheckKeyspaceLocked(ctx, ts.sourceKeyspace) + require.NoError(t, err) + + ts.cancelMigration(ctx, sm) + + // Expect the queries to be cleared + assert.Empty(t, env.tmc.vrQueries[100]) + assert.Empty(t, env.tmc.vrQueries[200]) +} + +func TestCancelMigration_SHARDS(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + workflowName := "wf1" + tableName := "t1" + + sourceKeyspace := &testKeyspace{ + KeyspaceName: "sourceks", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "targetks", + ShardNames: []string{"0"}, + } + + schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ + tableName: { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: tableName, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + }, + }, + }, + } + + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + env.tmc.schema = schema + + ts, _, err := env.ws.getWorkflowState(ctx, targetKeyspace.KeyspaceName, workflowName) + require.NoError(t, err) + ts.migrationType = binlogdata.MigrationType_SHARDS + + sm, err := BuildStreamMigrator(ctx, ts, false, sqlparser.NewTestParser()) + require.NoError(t, err) + + env.tmc.expectVRQuery(100, "update /*vt+ ALLOW_UNSAFE_VREPLICATION_WRITE */ _vt.vreplication set state='Running', stop_pos=null, message='' where db_name='vt_sourceks' and workflow != 'wf1_reverse'", &sqltypes.Result{}) + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running', message='' where db_name='vt_targetks' and workflow='wf1'", &sqltypes.Result{}) + env.tmc.expectVRQuery(100, "delete from _vt.vreplication where db_name = 'vt_sourceks' and workflow = 'wf1_reverse'", &sqltypes.Result{}) + + ctx, _, err = env.ts.LockKeyspace(ctx, targetKeyspace.KeyspaceName, "test") + require.NoError(t, err) + + ctx, _, err = env.ts.LockKeyspace(ctx, sourceKeyspace.KeyspaceName, "test") + require.NoError(t, err) + + err = topo.CheckKeyspaceLocked(ctx, ts.targetKeyspace) + require.NoError(t, err) + + err = topo.CheckKeyspaceLocked(ctx, ts.sourceKeyspace) + require.NoError(t, err) + + ts.cancelMigration(ctx, sm) + + // Expect the queries to be cleared + assert.Empty(t, env.tmc.vrQueries[100]) + assert.Empty(t, env.tmc.vrQueries[200]) +} diff --git a/go/vt/vtctl/workflow/utils_test.go b/go/vt/vtctl/workflow/utils_test.go index eecbfd6269b..99850639ac5 100644 --- a/go/vt/vtctl/workflow/utils_test.go +++ b/go/vt/vtctl/workflow/utils_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/testfiles" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo" @@ -22,6 +23,7 @@ import ( "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topotools" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" "vitess.io/vitess/go/vt/proto/vtctldata" ) @@ -280,3 +282,77 @@ func TestValidateSourceTablesExist(t *testing.T) { }) } } + +func TestLegacyBuildTargets(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + workflowName := "wf1" + tableName := "t1" + + sourceKeyspace := &testKeyspace{ + KeyspaceName: "sourceks", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "targetks", + ShardNames: []string{"-80", "80-"}, + } + + schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ + tableName: { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: tableName, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + }, + }, + }, + } + + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + env.tmc.schema = schema + + result1 := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "id|source|message|cell|tablet_types|workflow_type|workflow_sub_type|defer_secondary_keys", + "int64|varchar|varchar|varchar|varchar|int64|int64|int64"), + "1|keyspace:\"source\" shard:\"-80\" filter:{rules:{match:\"t1\"} rules:{match:\"t2\"}}||||0|0|0", + ) + result2 := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "id|source|message|cell|tablet_types|workflow_type|workflow_sub_type|defer_secondary_keys", + "int64|varchar|varchar|varchar|varchar|int64|int64|int64"), + "1|keyspace:\"source\" shard:\"80-\" filter:{rules:{match:\"t1\"} rules:{match:\"t2\"}}||||0|0|0", + "2|keyspace:\"source\" shard:\"80-\" filter:{rules:{match:\"t3\"} rules:{match:\"t4\"}}||||0|0|0", + ) + env.tmc.expectVRQuery(200, "select id, source, message, cell, tablet_types, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where workflow='wf1' and db_name='vt_targetks'", result1) + env.tmc.expectVRQuery(210, "select id, source, message, cell, tablet_types, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where workflow='wf1' and db_name='vt_targetks'", result2) + + ti, err := LegacyBuildTargets(ctx, env.ts, env.tmc, targetKeyspace.KeyspaceName, workflowName, targetKeyspace.ShardNames) + require.NoError(t, err) + // Expect 2 targets as there are 2 target shards. + assert.Len(t, ti.Targets, 2) + + assert.NotNil(t, ti.Targets["-80"]) + assert.NotNil(t, ti.Targets["80-"]) + + t1 := ti.Targets["-80"] + t2 := ti.Targets["80-"] + assert.Len(t, t1.Sources, 1) + assert.Len(t, t2.Sources, 2) + assert.Len(t, t1.Sources[1].Filter.Rules, 2) + + assert.Equal(t, t1.Sources[1].Filter.Rules[0].Match, "t1") + assert.Equal(t, t1.Sources[1].Filter.Rules[1].Match, "t2") + assert.Equal(t, t1.Sources[1].Shard, "-80") + + assert.Len(t, t2.Sources[1].Filter.Rules, 2) + assert.Len(t, t2.Sources[2].Filter.Rules, 2) + + assert.Equal(t, t2.Sources[1].Shard, "80-") + assert.Equal(t, t2.Sources[2].Shard, "80-") + assert.Equal(t, t2.Sources[1].Filter.Rules[0].Match, "t1") + assert.Equal(t, t2.Sources[1].Filter.Rules[1].Match, "t2") + assert.Equal(t, t2.Sources[2].Filter.Rules[0].Match, "t3") + assert.Equal(t, t2.Sources[2].Filter.Rules[1].Match, "t4") +} From 489fd05eda9e90f8583a2e81e8fa5c7417759fe8 Mon Sep 17 00:00:00 2001 From: jwang <121262788+jwangace@users.noreply.github.com> Date: Mon, 27 Jan 2025 15:36:39 -0800 Subject: [PATCH 02/68] --consolidator-query-waiter-cap to set the max number of waiter for consolidated query (#17244) Signed-off-by: Jun Wang Co-authored-by: Jun Wang --- changelog/22.0/22.0.0/summary.md | 2 ++ go/flags/endtoend/vtcombo.txt | 1 + go/flags/endtoend/vttablet.txt | 1 + go/sync2/consolidator.go | 10 +++++- go/sync2/consolidator_test.go | 31 +++++++++++++++++++ go/sync2/fake_consolidator.go | 5 +++ go/vt/vttablet/tabletserver/query_executor.go | 12 ++++--- .../vttablet/tabletserver/tabletenv/config.go | 2 ++ 8 files changed, 59 insertions(+), 5 deletions(-) diff --git a/changelog/22.0/22.0.0/summary.md b/changelog/22.0/22.0.0/summary.md index b2c5c029851..7375077dced 100644 --- a/changelog/22.0/22.0.0/summary.md +++ b/changelog/22.0/22.0.0/summary.md @@ -147,6 +147,8 @@ The `querylog-mode` setting can be configured to `error` to log only queries tha While the flag will continue to accept float values (interpreted as seconds) for backward compatibility, **float inputs are deprecated** and will be removed in a future release. +- `--consolidator-query-waiter-cap` flag to set the maximum number of clients allowed to wait on the consolidator. The default value is set to 0 for unlimited wait. Users can adjust this value based on the performance of VTTablet to avoid excessive memory usage and the risk of being OOMKilled, particularly in Kubernetes deployments. + ### `--topo_read_concurrency` behaviour changes The `--topo_read_concurrency` flag was added to all components that access the topology and the provided limit is now applied separately for each global or local cell _(default `32`)_. diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 561f6048ce6..3ae0cbc9b77 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -44,6 +44,7 @@ Flags: --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). + --consolidator-query-waiter-cap int Configure the maximum number of clients allowed to wait on the consolidator. --consolidator-stream-query-size int Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152) --consolidator-stream-total-size int Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator. (default 134217728) --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 398d10afd7c..0a9a0ef99ce 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -78,6 +78,7 @@ Flags: --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). + --consolidator-query-waiter-cap int Configure the maximum number of clients allowed to wait on the consolidator. --consolidator-stream-query-size int Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152) --consolidator-stream-total-size int Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator. (default 134217728) --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/sync2/consolidator.go b/go/sync2/consolidator.go index 401daaef1f1..df900625abb 100644 --- a/go/sync2/consolidator.go +++ b/go/sync2/consolidator.go @@ -40,6 +40,7 @@ type PendingResult interface { SetResult(*sqltypes.Result) Result() *sqltypes.Result Wait() + AddWaiterCounter(int64) *int64 } type consolidator struct { @@ -77,6 +78,7 @@ func (co *consolidator) Create(query string) (PendingResult, bool) { defer co.mu.Unlock() var r *pendingResult if r, ok := co.queries[query]; ok { + r.AddWaiterCounter(1) return r, false } r = &pendingResult{consolidator: co, query: query} @@ -122,17 +124,23 @@ func (rs *pendingResult) Wait() { rs.executing.RLock() } +func (rs *pendingResult) AddWaiterCounter(c int64) *int64 { + atomic.AddInt64(rs.consolidator.totalWaiterCount, c) + return rs.consolidator.totalWaiterCount +} + // ConsolidatorCache is a thread-safe object used for counting how often recent // queries have been consolidated. // It is also used by the txserializer package to count how often transactions // have been queued and had to wait because they targeted the same row (range). type ConsolidatorCache struct { *cache.LRUCache[*ccount] + totalWaiterCount *int64 } // NewConsolidatorCache creates a new cache with the given capacity. func NewConsolidatorCache(capacity int64) *ConsolidatorCache { - return &ConsolidatorCache{cache.NewLRUCache[*ccount](capacity)} + return &ConsolidatorCache{cache.NewLRUCache[*ccount](capacity), new(int64)} } // Record increments the count for "query" by 1. diff --git a/go/sync2/consolidator_test.go b/go/sync2/consolidator_test.go index 132a253ba29..5437bb335a6 100644 --- a/go/sync2/consolidator_test.go +++ b/go/sync2/consolidator_test.go @@ -18,11 +18,42 @@ package sync2 import ( "reflect" + "sync" "testing" "vitess.io/vitess/go/sqltypes" ) +func TestAddWaiterCount(t *testing.T) { + con := NewConsolidator() + sql := "select * from SomeTable" + pr, _ := con.Create(sql) + var wgAdd sync.WaitGroup + var wgSub sync.WaitGroup + + var concurrent = 1000 + + for i := 0; i < concurrent; i++ { + wgAdd.Add(1) + wgSub.Add(1) + go func() { + defer wgAdd.Done() + pr.AddWaiterCounter(1) + }() + go func() { + defer wgSub.Done() + pr.AddWaiterCounter(-1) + }() + } + + wgAdd.Wait() + wgSub.Wait() + + if *pr.AddWaiterCounter(0) != 0 { + t.Fatalf("Expect 0 totalWaiterCount but got: %v", *pr.AddWaiterCounter(0)) + } +} + func TestConsolidator(t *testing.T) { con := NewConsolidator() sql := "select * from SomeTable" diff --git a/go/sync2/fake_consolidator.go b/go/sync2/fake_consolidator.go index 64c59e78a5a..aadee1d37ce 100644 --- a/go/sync2/fake_consolidator.go +++ b/go/sync2/fake_consolidator.go @@ -112,3 +112,8 @@ func (fr *FakePendingResult) SetResult(result *sqltypes.Result) { func (fr *FakePendingResult) Wait() { fr.WaitCalls++ } + +// AddWaiterCounter is currently a no-op. +func (fr *FakePendingResult) AddWaiterCounter(int64) *int64 { + return new(int64) +} diff --git a/go/vt/vttablet/tabletserver/query_executor.go b/go/vt/vttablet/tabletserver/query_executor.go index e4a165960fd..755394723f3 100644 --- a/go/vt/vttablet/tabletserver/query_executor.go +++ b/go/vt/vttablet/tabletserver/query_executor.go @@ -718,10 +718,14 @@ func (qre *QueryExecutor) execSelect() (*sqltypes.Result, error) { q.SetErr(err) } } else { - qre.logStats.QuerySources |= tabletenv.QuerySourceConsolidator - startTime := time.Now() - q.Wait() - qre.tsv.stats.WaitTimings.Record("Consolidations", startTime) + waiterCap := qre.tsv.config.ConsolidatorQueryWaiterCap + if waiterCap == 0 || *q.AddWaiterCounter(0) <= waiterCap { + qre.logStats.QuerySources |= tabletenv.QuerySourceConsolidator + startTime := time.Now() + q.Wait() + qre.tsv.stats.WaitTimings.Record("Consolidations", startTime) + } + q.AddWaiterCounter(-1) } if q.Err() != nil { return nil, q.Err() diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 08bf0b1d7c8..ddab935d393 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -198,6 +198,7 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) { fs.Int64Var(¤tConfig.ConsolidatorStreamQuerySize, "consolidator-stream-query-size", defaultConfig.ConsolidatorStreamQuerySize, "Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator.") fs.Int64Var(¤tConfig.ConsolidatorStreamTotalSize, "consolidator-stream-total-size", defaultConfig.ConsolidatorStreamTotalSize, "Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator.") + fs.Int64Var(¤tConfig.ConsolidatorQueryWaiterCap, "consolidator-query-waiter-cap", 0, "Configure the maximum number of clients allowed to wait on the consolidator.") fs.DurationVar(&healthCheckInterval, "health_check_interval", defaultConfig.Healthcheck.Interval, "Interval between health checks") fs.DurationVar(°radedThreshold, "degraded_threshold", defaultConfig.Healthcheck.DegradedThreshold, "replication lag after which a replica is considered degraded") fs.DurationVar(&unhealthyThreshold, "unhealthy_threshold", defaultConfig.Healthcheck.UnhealthyThreshold, "replication lag after which a replica is considered unhealthy") @@ -324,6 +325,7 @@ type TabletConfig struct { StreamBufferSize int `json:"streamBufferSize,omitempty"` ConsolidatorStreamTotalSize int64 `json:"consolidatorStreamTotalSize,omitempty"` ConsolidatorStreamQuerySize int64 `json:"consolidatorStreamQuerySize,omitempty"` + ConsolidatorQueryWaiterCap int64 `json:"consolidatorMaxQueryWait,omitempty"` QueryCacheMemory int64 `json:"queryCacheMemory,omitempty"` QueryCacheDoorkeeper bool `json:"queryCacheDoorkeeper,omitempty"` SchemaReloadInterval time.Duration `json:"schemaReloadIntervalSeconds,omitempty"` From 8921bce64b732c48ca6bb52037fa9b2796e36d1d Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Tue, 28 Jan 2025 11:59:34 +0530 Subject: [PATCH 03/68] Support KeyRange in `--clusters_to_watch` flag (#17604) Signed-off-by: Manan Gupta --- changelog/22.0/22.0.0/summary.md | 6 + go/flags/endtoend/vtorc.txt | 2 +- go/vt/key/key.go | 8 + go/vt/topo/shard_test.go | 8 + go/vt/vtorc/logic/keyspace_shard_discovery.go | 26 +-- .../logic/keyspace_shard_discovery_test.go | 5 +- go/vt/vtorc/logic/tablet_discovery.go | 90 ++++---- go/vt/vtorc/logic/tablet_discovery_test.go | 200 +++++++++++++++--- go/vt/vtorc/logic/vtorc.go | 6 - 9 files changed, 258 insertions(+), 93 deletions(-) diff --git a/changelog/22.0/22.0.0/summary.md b/changelog/22.0/22.0.0/summary.md index 7375077dced..f773df2b10c 100644 --- a/changelog/22.0/22.0.0/summary.md +++ b/changelog/22.0/22.0.0/summary.md @@ -15,6 +15,7 @@ - **[Stalled Disk Recovery in VTOrc](#stall-disk-recovery)** - **[Update default MySQL version to 8.0.40](#mysql-8-0-40)** - **[Update lite images to Debian Bookworm](#debian-bookworm)** + - **[KeyRanges in `--clusters_to_watch` in VTOrc](#key-range-vtorc)** - **[Support for Filtering Query logs on Error](#query-logs)** - **[Minor Changes](#minor-changes)** - **[VTTablet Flags](#flags-vttablet)** @@ -135,6 +136,11 @@ This is the last time this will be needed in the `8.0.x` series, as starting wit The base system now uses Debian Bookworm instead of Debian Bullseye for the `vitess/lite` images. This change was brought by [Pull Request #17552]. +### KeyRanges in `--clusters_to_watch` in VTOrc +VTOrc now supports specifying keyranges in the `--clusters_to_watch` flag. This means that there is no need to restart a VTOrc instance with a different flag value when you reshard a keyspace. +For example, if a VTOrc is configured to watch `ks/-80`, then it would watch all the shards that fall under the keyrange `-80`. If a reshard is performed and `-80` is split into new shards `-40` and `40-80`, the VTOrc instance will automatically start watching the new shards without needing a restart. In the previous logic, specifying `ks/-80` for the flag would mean that VTOrc would watch only 1 (or no) shard. In the new system, since we interpret `-80` as a key range, it can watch multiple shards as described in the example. +Users can continue to specify exact keyranges. The new feature is backward compatible. + ### Support for Filtering Query logs on Error The `querylog-mode` setting can be configured to `error` to log only queries that result in errors. This option is supported in both VTGate and VTTablet. diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index ca8083709e5..57eb907cf4d 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -24,7 +24,7 @@ Flags: --bind-address string Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system. --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified --change-tablets-with-errant-gtid-to-drained Whether VTOrc should be changing the type of tablets with errant GTIDs to DRAINED - --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" + --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/keyranges that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") diff --git a/go/vt/key/key.go b/go/vt/key/key.go index 89d956bd433..82852daa16e 100644 --- a/go/vt/key/key.go +++ b/go/vt/key/key.go @@ -95,6 +95,14 @@ func NewKeyRange(start []byte, end []byte) *topodatapb.KeyRange { return &topodatapb.KeyRange{Start: start, End: end} } +// NewCompleteKeyRange returns a complete key range. +func NewCompleteKeyRange() *topodatapb.KeyRange { + return &topodatapb.KeyRange{ + Start: nil, + End: nil, + } +} + // KeyRangeAdd adds two adjacent KeyRange values (in any order) into a single value. If the values are not adjacent, // it returns false. func KeyRangeAdd(a, b *topodatapb.KeyRange) (*topodatapb.KeyRange, bool) { diff --git a/go/vt/topo/shard_test.go b/go/vt/topo/shard_test.go index 6bd4aae5b62..915bcd18e3c 100644 --- a/go/vt/topo/shard_test.go +++ b/go/vt/topo/shard_test.go @@ -323,6 +323,14 @@ func TestValidateShardName(t *testing.T) { }, valid: true, }, + { + name: "-", + expectedRange: &topodatapb.KeyRange{ + Start: []byte{}, + End: []byte{}, + }, + valid: true, + }, { name: "40-80", expectedRange: &topodatapb.KeyRange{ diff --git a/go/vt/vtorc/logic/keyspace_shard_discovery.go b/go/vt/vtorc/logic/keyspace_shard_discovery.go index 0dd17cb65fd..8115e614418 100644 --- a/go/vt/vtorc/logic/keyspace_shard_discovery.go +++ b/go/vt/vtorc/logic/keyspace_shard_discovery.go @@ -18,10 +18,10 @@ package logic import ( "context" - "sort" - "strings" "sync" + "golang.org/x/exp/maps" + "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo" @@ -31,7 +31,7 @@ import ( // RefreshAllKeyspacesAndShards reloads the keyspace and shard information for the keyspaces that vtorc is concerned with. func RefreshAllKeyspacesAndShards(ctx context.Context) error { var keyspaces []string - if len(clustersToWatch) == 0 { // all known keyspaces + if len(shardsToWatch) == 0 { // all known keyspaces ctx, cancel := context.WithTimeout(ctx, topo.RemoteOperationTimeout) defer cancel() var err error @@ -41,26 +41,10 @@ func RefreshAllKeyspacesAndShards(ctx context.Context) error { return err } } else { - // Parse input and build list of keyspaces - for _, ks := range clustersToWatch { - if strings.Contains(ks, "/") { - // This is a keyspace/shard specification - input := strings.Split(ks, "/") - keyspaces = append(keyspaces, input[0]) - } else { - // Assume this is a keyspace - keyspaces = append(keyspaces, ks) - } - } - if len(keyspaces) == 0 { - log.Errorf("Found no keyspaces for input: %+v", clustersToWatch) - return nil - } + // Get keyspaces to watch from the list of known keyspaces. + keyspaces = maps.Keys(shardsToWatch) } - // Sort the list of keyspaces. - // The list can have duplicates because the input to clusters to watch may have multiple shards of the same keyspace - sort.Strings(keyspaces) refreshCtx, refreshCancel := context.WithTimeout(ctx, topo.RemoteOperationTimeout) defer refreshCancel() var wg sync.WaitGroup diff --git a/go/vt/vtorc/logic/keyspace_shard_discovery_test.go b/go/vt/vtorc/logic/keyspace_shard_discovery_test.go index 8218af45db6..f05295416d0 100644 --- a/go/vt/vtorc/logic/keyspace_shard_discovery_test.go +++ b/go/vt/vtorc/logic/keyspace_shard_discovery_test.go @@ -93,6 +93,8 @@ func TestRefreshAllKeyspaces(t *testing.T) { // Set clusters to watch to only watch ks1 and ks3 onlyKs1and3 := []string{"ks1/-80", "ks3/-80", "ks3/80-"} clustersToWatch = onlyKs1and3 + err := initializeShardsToWatch() + require.NoError(t, err) require.NoError(t, RefreshAllKeyspacesAndShards(context.Background())) // Verify that we only have ks1 and ks3 in vtorc's db. @@ -106,6 +108,8 @@ func TestRefreshAllKeyspaces(t *testing.T) { // Set clusters to watch to watch all keyspaces clustersToWatch = nil + err = initializeShardsToWatch() + require.NoError(t, err) // Change the durability policy of ks1 reparenttestutil.SetKeyspaceDurability(ctx, t, ts, "ks1", policy.DurabilitySemiSync) require.NoError(t, RefreshAllKeyspacesAndShards(context.Background())) @@ -119,7 +123,6 @@ func TestRefreshAllKeyspaces(t *testing.T) { verifyPrimaryAlias(t, "ks3", "80-", "zone_ks3-0000000101", "") verifyKeyspaceInfo(t, "ks4", keyspaceDurabilityTest, "") verifyPrimaryAlias(t, "ks4", "80-", "zone_ks4-0000000101", "") - } func TestRefreshKeyspace(t *testing.T) { diff --git a/go/vt/vtorc/logic/tablet_discovery.go b/go/vt/vtorc/logic/tablet_discovery.go index eb10bb2a667..c5c23df0cd0 100644 --- a/go/vt/vtorc/logic/tablet_discovery.go +++ b/go/vt/vtorc/logic/tablet_discovery.go @@ -32,6 +32,7 @@ import ( "google.golang.org/protobuf/proto" "vitess.io/vitess/go/vt/external/golib/sqlutils" + "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo" @@ -48,8 +49,10 @@ var ( clustersToWatch []string shutdownWaitTime = 30 * time.Second shardsLockCounter int32 - shardsToWatch map[string][]string - shardsToWatchMu sync.Mutex + // shardsToWatch is a map storing the shards for a given keyspace that need to be watched. + // We store the key range for all the shards that we want to watch. + // This is populated by parsing `--clusters_to_watch` flag. + shardsToWatch map[string][]*topodatapb.KeyRange // ErrNoPrimaryTablet is a fixed error message. ErrNoPrimaryTablet = errors.New("no primary tablet found") @@ -57,18 +60,18 @@ var ( // RegisterFlags registers the flags required by VTOrc func RegisterFlags(fs *pflag.FlagSet) { - fs.StringSliceVar(&clustersToWatch, "clusters_to_watch", clustersToWatch, "Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: \"ks1,ks2/-80\"") + fs.StringSliceVar(&clustersToWatch, "clusters_to_watch", clustersToWatch, "Comma-separated list of keyspaces or keyspace/keyranges that this instance will monitor and repair. Defaults to all clusters in the topology. Example: \"ks1,ks2/-80\"") fs.DurationVar(&shutdownWaitTime, "shutdown_wait_time", shutdownWaitTime, "Maximum time to wait for VTOrc to release all the locks that it is holding before shutting down on SIGTERM") } -// updateShardsToWatch parses the --clusters_to_watch flag-value +// initializeShardsToWatch parses the --clusters_to_watch flag-value // into a map of keyspace/shards. -func updateShardsToWatch() { +func initializeShardsToWatch() error { + shardsToWatch = make(map[string][]*topodatapb.KeyRange) if len(clustersToWatch) == 0 { - return + return nil } - newShardsToWatch := make(map[string][]string, 0) for _, ks := range clustersToWatch { if strings.Contains(ks, "/") && !strings.HasSuffix(ks, "/") { // Validate keyspace/shard parses. @@ -77,34 +80,50 @@ func updateShardsToWatch() { log.Errorf("Could not parse keyspace/shard %q: %+v", ks, err) continue } - newShardsToWatch[k] = append(newShardsToWatch[k], s) + if !key.IsValidKeyRange(s) { + return fmt.Errorf("invalid key range %q while parsing clusters to watch", s) + } + // Parse the shard name into key range value. + keyRanges, err := key.ParseShardingSpec(s) + if err != nil { + return fmt.Errorf("could not parse shard name %q: %+v", s, err) + } + shardsToWatch[k] = append(shardsToWatch[k], keyRanges...) } else { - ctx, cancel := context.WithTimeout(context.Background(), topo.RemoteOperationTimeout) - defer cancel() - // Assume this is a keyspace and find all shards in keyspace. // Remove trailing slash if exists. ks = strings.TrimSuffix(ks, "/") - shards, err := ts.GetShardNames(ctx, ks) - if err != nil { - // Log the err and continue. - log.Errorf("Error fetching shards for keyspace: %v", ks) - continue - } - if len(shards) == 0 { - log.Errorf("Topo has no shards for ks: %v", ks) - continue - } - newShardsToWatch[ks] = shards + // We store the entire range of key range if nothing is specified. + shardsToWatch[ks] = []*topodatapb.KeyRange{key.NewCompleteKeyRange()} } } - if len(newShardsToWatch) == 0 { - log.Error("No keyspace/shards to watch") - return + + if len(shardsToWatch) == 0 { + log.Error("No keyspace/shards to watch, watching all keyspaces") } + return nil +} - shardsToWatchMu.Lock() - defer shardsToWatchMu.Unlock() - shardsToWatch = newShardsToWatch +// shouldWatchTablet checks if the given tablet is part of the watch list. +func shouldWatchTablet(tablet *topodatapb.Tablet) bool { + // If we are watching all keyspaces, then we want to watch this tablet too. + if len(shardsToWatch) == 0 { + return true + } + shardRanges, ok := shardsToWatch[tablet.GetKeyspace()] + // If we don't have the keyspace in our map, then this tablet + // doesn't need to be watched. + if !ok { + return false + } + // Get the tablet's key range, and check if + // it is part of the shard ranges we are watching. + kr := tablet.GetKeyRange() + for _, shardRange := range shardRanges { + if key.KeyRangeContainsKeyRange(shardRange, kr) { + return true + } + } + return false } // OpenTabletDiscovery opens the vitess topo if enables and returns a ticker @@ -117,7 +136,10 @@ func OpenTabletDiscovery() <-chan time.Time { log.Error(err) } // Parse --clusters_to_watch into a filter. - updateShardsToWatch() + err := initializeShardsToWatch() + if err != nil { + log.Fatalf("Error parsing --clusters-to-watch: %v", err) + } // We refresh all information from the topo once before we start the ticks to do // it on a timer. ctx, cancel := context.WithTimeout(context.Background(), topo.RemoteOperationTimeout) @@ -179,16 +201,10 @@ func refreshTabletsUsing(ctx context.Context, loader func(tabletAlias string), f // Filter tablets that should not be watched using shardsToWatch map. matchedTablets := make([]*topo.TabletInfo, 0, len(tablets)) func() { - shardsToWatchMu.Lock() - defer shardsToWatchMu.Unlock() for _, t := range tablets { - if len(shardsToWatch) > 0 { - _, ok := shardsToWatch[t.Tablet.Keyspace] - if !ok || !slices.Contains(shardsToWatch[t.Tablet.Keyspace], t.Tablet.Shard) { - continue // filter - } + if shouldWatchTablet(t.Tablet) { + matchedTablets = append(matchedTablets, t) } - matchedTablets = append(matchedTablets, t) } }() diff --git a/go/vt/vtorc/logic/tablet_discovery_test.go b/go/vt/vtorc/logic/tablet_discovery_test.go index 54284e8a017..4514ef81724 100644 --- a/go/vt/vtorc/logic/tablet_discovery_test.go +++ b/go/vt/vtorc/logic/tablet_discovery_test.go @@ -30,6 +30,7 @@ import ( "google.golang.org/protobuf/proto" "vitess.io/vitess/go/vt/external/golib/sqlutils" + "vitess.io/vitess/go/vt/key" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/proto/vttime" "vitess.io/vitess/go/vt/topo" @@ -102,60 +103,200 @@ var ( } ) -func TestUpdateShardsToWatch(t *testing.T) { +func TestShouldWatchTablet(t *testing.T) { oldClustersToWatch := clustersToWatch - oldTs := ts defer func() { clustersToWatch = oldClustersToWatch shardsToWatch = nil - ts = oldTs }() - // Create a memory topo-server and create the keyspace and shard records - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + testCases := []struct { + in []string + tablet *topodatapb.Tablet + expectedShouldWatch bool + }{ + { + in: []string{}, + tablet: &topodatapb.Tablet{ + Keyspace: keyspace, + Shard: shard, + }, + expectedShouldWatch: true, + }, + { + in: []string{keyspace}, + tablet: &topodatapb.Tablet{ + Keyspace: keyspace, + Shard: shard, + }, + expectedShouldWatch: true, + }, + { + in: []string{keyspace + "/-"}, + tablet: &topodatapb.Tablet{ + Keyspace: keyspace, + Shard: shard, + }, + expectedShouldWatch: true, + }, + { + in: []string{keyspace + "/" + shard}, + tablet: &topodatapb.Tablet{ + Keyspace: keyspace, + Shard: shard, + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks/-70", "ks/70-"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x50}, []byte{0x70}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks/-70", "ks/70-"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x40}, []byte{0x50}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks/-70", "ks/70-"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x70}, []byte{0x90}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks/-70", "ks/70-"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x60}, []byte{0x90}), + }, + expectedShouldWatch: false, + }, + { + in: []string{"ks/50-70"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x50}, []byte{0x70}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks2/-70", "ks2/70-", "unknownKs/-", "ks/-80"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x60}, []byte{0x80}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks2/-70", "ks2/70-", "unknownKs/-", "ks/-80"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x80}, []byte{0x90}), + }, + expectedShouldWatch: false, + }, + { + in: []string{"ks2/-70", "ks2/70-", "unknownKs/-", "ks/-80"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x90}, []byte{0xa0}), + }, + expectedShouldWatch: false, + }, + } - ts = memorytopo.NewServer(ctx, cell1) - _, err := ts.GetOrCreateShard(context.Background(), keyspace, shard) - require.NoError(t, err) + for _, tt := range testCases { + t.Run(fmt.Sprintf("%v-Tablet-%v-%v", strings.Join(tt.in, ","), tt.tablet.GetKeyspace(), tt.tablet.GetShard()), func(t *testing.T) { + clustersToWatch = tt.in + err := initializeShardsToWatch() + require.NoError(t, err) + assert.Equal(t, tt.expectedShouldWatch, shouldWatchTablet(tt.tablet)) + }) + } +} + +// TestInitializeShardsToWatch tests that we initialize the shardsToWatch map correctly +// using the `--clusters_to_watch` flag. +func TestInitializeShardsToWatch(t *testing.T) { + oldClustersToWatch := clustersToWatch + defer func() { + clustersToWatch = oldClustersToWatch + shardsToWatch = nil + }() testCases := []struct { - in []string - expected map[string][]string + in []string + expected map[string][]*topodatapb.KeyRange + expectedErr string }{ { in: []string{}, - expected: nil, + expected: map[string][]*topodatapb.KeyRange{}, }, { - in: []string{""}, - expected: map[string][]string{}, + in: []string{"unknownKs"}, + expected: map[string][]*topodatapb.KeyRange{ + "unknownKs": { + key.NewCompleteKeyRange(), + }, + }, }, { in: []string{"test/-"}, - expected: map[string][]string{ - "test": {"-"}, + expected: map[string][]*topodatapb.KeyRange{ + "test": { + key.NewCompleteKeyRange(), + }, + }, + }, + { + in: []string{"test/324"}, + expectedErr: `invalid key range "324" while parsing clusters to watch`, + }, + { + in: []string{"test/0"}, + expected: map[string][]*topodatapb.KeyRange{ + "test": { + key.NewCompleteKeyRange(), + }, }, }, { in: []string{"test/-", "test2/-80", "test2/80-"}, - expected: map[string][]string{ - "test": {"-"}, - "test2": {"-80", "80-"}, + expected: map[string][]*topodatapb.KeyRange{ + "test": { + key.NewCompleteKeyRange(), + }, + "test2": { + key.NewKeyRange(nil, []byte{0x80}), + key.NewKeyRange([]byte{0x80}, nil), + }, }, }, { - // confirm shards fetch from topo + // known keyspace in: []string{keyspace}, - expected: map[string][]string{ - keyspace: {shard}, + expected: map[string][]*topodatapb.KeyRange{ + keyspace: { + key.NewCompleteKeyRange(), + }, }, }, { - // confirm shards fetch from topo when keyspace has trailing-slash + // keyspace with trailing-slash in: []string{keyspace + "/"}, - expected: map[string][]string{ - keyspace: {shard}, + expected: map[string][]*topodatapb.KeyRange{ + keyspace: { + key.NewCompleteKeyRange(), + }, }, }, } @@ -163,10 +304,15 @@ func TestUpdateShardsToWatch(t *testing.T) { for _, testCase := range testCases { t.Run(strings.Join(testCase.in, ","), func(t *testing.T) { defer func() { - shardsToWatch = make(map[string][]string, 0) + shardsToWatch = make(map[string][]*topodatapb.KeyRange, 0) }() clustersToWatch = testCase.in - updateShardsToWatch() + err := initializeShardsToWatch() + if testCase.expectedErr != "" { + require.EqualError(t, err, testCase.expectedErr) + return + } + require.NoError(t, err) require.Equal(t, testCase.expected, shardsToWatch) }) } diff --git a/go/vt/vtorc/logic/vtorc.go b/go/vt/vtorc/logic/vtorc.go index 1fde6e31c0d..5ac5af50d47 100644 --- a/go/vt/vtorc/logic/vtorc.go +++ b/go/vt/vtorc/logic/vtorc.go @@ -326,12 +326,6 @@ func refreshAllInformation(ctx context.Context) error { return RefreshAllKeyspacesAndShards(ctx) }) - // Refresh shards to watch. - eg.Go(func() error { - updateShardsToWatch() - return nil - }) - // Refresh all tablets. eg.Go(func() error { return refreshAllTablets(ctx) From c2f4dcf80a86e092ca7ce43bd3eb5d5eb66cc999 Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Tue, 28 Jan 2025 11:59:45 +0530 Subject: [PATCH 04/68] Refactor Disk Stall implementation and mark tablet not serving if disk is stalled (#17624) Signed-off-by: Manan Gupta --- .../replicationdata/replicationdata.pb.go | 30 ++++++--- .../replicationdata_vtproto.pb.go | 36 +++++++++++ go/vt/vtorc/inst/instance_dao.go | 7 ++- .../vttablet/tabletmanager/rpc_replication.go | 12 ++-- go/vt/vttablet/tabletmanager/tm_init.go | 12 +--- go/vt/vttablet/tabletserver/controller.go | 3 + .../disk_health_monitor.go | 39 +++++++++++- .../disk_health_monitor_test.go | 18 +++++- go/vt/vttablet/tabletserver/state_manager.go | 3 +- .../tabletserver/state_manager_test.go | 62 +++++++++++++------ go/vt/vttablet/tabletserver/tabletserver.go | 40 +++++++----- go/vt/vttablet/tabletservermock/controller.go | 6 ++ proto/replicationdata.proto | 1 + web/vtadmin/src/proto/vtadmin.d.ts | 6 ++ web/vtadmin/src/proto/vtadmin.js | 23 +++++++ 15 files changed, 232 insertions(+), 66 deletions(-) rename go/vt/vttablet/{tabletmanager => tabletserver}/disk_health_monitor.go (67%) rename go/vt/vttablet/{tabletmanager => tabletserver}/disk_health_monitor_test.go (85%) diff --git a/go/vt/proto/replicationdata/replicationdata.pb.go b/go/vt/proto/replicationdata/replicationdata.pb.go index d5462e1ea2b..bb973577d55 100644 --- a/go/vt/proto/replicationdata/replicationdata.pb.go +++ b/go/vt/proto/replicationdata/replicationdata.pb.go @@ -509,6 +509,7 @@ type FullStatus struct { SemiSyncWaitForReplicaCount uint32 `protobuf:"varint,20,opt,name=semi_sync_wait_for_replica_count,json=semiSyncWaitForReplicaCount,proto3" json:"semi_sync_wait_for_replica_count,omitempty"` SuperReadOnly bool `protobuf:"varint,21,opt,name=super_read_only,json=superReadOnly,proto3" json:"super_read_only,omitempty"` ReplicationConfiguration *Configuration `protobuf:"bytes,22,opt,name=replication_configuration,json=replicationConfiguration,proto3" json:"replication_configuration,omitempty"` + DiskStalled bool `protobuf:"varint,23,opt,name=disk_stalled,json=diskStalled,proto3" json:"disk_stalled,omitempty"` } func (x *FullStatus) Reset() { @@ -695,6 +696,13 @@ func (x *FullStatus) GetReplicationConfiguration() *Configuration { return nil } +func (x *FullStatus) GetDiskStalled() bool { + if x != nil { + return x.DiskStalled + } + return false +} + var File_replicationdata_proto protoreflect.FileDescriptor var file_replicationdata_proto_rawDesc = []byte{ @@ -782,7 +790,7 @@ var file_replicationdata_proto_rawDesc = []byte{ 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x55, 0x75, 0x69, 0x64, 0x22, 0xc8, 0x08, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, + 0x55, 0x75, 0x69, 0x64, 0x22, 0xeb, 0x08, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x18, @@ -850,15 +858,17 @@ var file_replicationdata_proto_rawDesc = []byte{ 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x18, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2a, - 0x3b, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4f, 0x41, 0x4e, 0x44, 0x53, - 0x51, 0x4c, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x49, 0x4f, - 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x42, 0x2e, 0x5a, 0x2c, - 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, - 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x21, 0x0a, 0x0c, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x18, + 0x17, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x6b, 0x53, 0x74, 0x61, 0x6c, 0x6c, + 0x65, 0x64, 0x2a, 0x3b, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4f, 0x41, + 0x4e, 0x44, 0x53, 0x51, 0x4c, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x10, 0x00, 0x12, 0x10, 0x0a, + 0x0c, 0x49, 0x4f, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x42, + 0x2e, 0x5a, 0x2c, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, + 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, + 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x64, 0x61, 0x74, 0x61, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/go/vt/proto/replicationdata/replicationdata_vtproto.pb.go b/go/vt/proto/replicationdata/replicationdata_vtproto.pb.go index b3d638a1327..a515397c065 100644 --- a/go/vt/proto/replicationdata/replicationdata_vtproto.pb.go +++ b/go/vt/proto/replicationdata/replicationdata_vtproto.pb.go @@ -142,6 +142,7 @@ func (m *FullStatus) CloneVT() *FullStatus { r.SemiSyncWaitForReplicaCount = m.SemiSyncWaitForReplicaCount r.SuperReadOnly = m.SuperReadOnly r.ReplicationConfiguration = m.ReplicationConfiguration.CloneVT() + r.DiskStalled = m.DiskStalled if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -552,6 +553,18 @@ func (m *FullStatus) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.DiskStalled { + i-- + if m.DiskStalled { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xb8 + } if m.ReplicationConfiguration != nil { size, err := m.ReplicationConfiguration.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -975,6 +988,9 @@ func (m *FullStatus) SizeVT() (n int) { l = m.ReplicationConfiguration.SizeVT() n += 2 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.DiskStalled { + n += 3 + } n += len(m.unknownFields) return n } @@ -2551,6 +2567,26 @@ func (m *FullStatus) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 23: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DiskStalled", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.DiskStalled = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index f92a15079dd..9e35e6e3e0b 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -206,9 +206,10 @@ func ReadTopologyInstanceBufferable(tabletAlias string, latency *stopwatch.Named fs, err = fullStatus(tabletAlias) if err != nil { - if config.GetStalledDiskPrimaryRecovery() && strings.Contains(err.Error(), "stalled disk") { - stalledDisk = true - } + goto Cleanup + } + if config.GetStalledDiskPrimaryRecovery() && fs.DiskStalled { + stalledDisk = true goto Cleanup } partialSuccess = true // We at least managed to read something from the server. diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index b27b25d87c6..dec94ee6f16 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -18,7 +18,6 @@ package tabletmanager import ( "context" - "errors" "fmt" "runtime" "strings" @@ -62,10 +61,13 @@ func (tm *TabletManager) FullStatus(ctx context.Context) (*replicationdatapb.Ful return nil, err } - // Return error if the disk is stalled or rejecting writes. - // Noop by default, must be enabled with the flag "disk-write-dir". - if tm.dhMonitor.IsDiskStalled() { - return nil, errors.New("stalled disk") + // Return if the disk is stalled or rejecting writes. + // If the disk is stalled, we can't be sure if reads will go through + // or not, so we should not run any reads either. + if tm.QueryServiceControl.IsDiskStalled() { + return &replicationdatapb.FullStatus{ + DiskStalled: true, + }, nil } // Server ID - "select @@global.server_id" diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index c22ea0a6e51..fbef04de357 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -95,11 +95,8 @@ var ( skipBuildInfoTags = "/.*/" initTags flagutil.StringMapValue - initTimeout = 1 * time.Minute - mysqlShutdownTimeout = mysqlctl.DefaultShutdownTimeout - stalledDiskWriteDir = "" - stalledDiskWriteTimeout = 30 * time.Second - stalledDiskWriteInterval = 5 * time.Second + initTimeout = 1 * time.Minute + mysqlShutdownTimeout = mysqlctl.DefaultShutdownTimeout ) func registerInitFlags(fs *pflag.FlagSet) { @@ -112,9 +109,6 @@ func registerInitFlags(fs *pflag.FlagSet) { fs.Var(&initTags, "init_tags", "(init parameter) comma separated list of key:value pairs used to tag the tablet") fs.DurationVar(&initTimeout, "init_timeout", initTimeout, "(init parameter) timeout to use for the init phase.") fs.DurationVar(&mysqlShutdownTimeout, "mysql-shutdown-timeout", mysqlShutdownTimeout, "timeout to use when MySQL is being shut down.") - fs.StringVar(&stalledDiskWriteDir, "disk-write-dir", stalledDiskWriteDir, "if provided, tablet will attempt to write a file to this directory to check if the disk is stalled") - fs.DurationVar(&stalledDiskWriteTimeout, "disk-write-timeout", stalledDiskWriteTimeout, "if writes exceed this duration, the disk is considered stalled") - fs.DurationVar(&stalledDiskWriteInterval, "disk-write-interval", stalledDiskWriteInterval, "how often to write to the disk to check whether it is stalled") } var ( @@ -170,7 +164,6 @@ type TabletManager struct { VREngine *vreplication.Engine VDiffEngine *vdiff.Engine Env *vtenv.Environment - dhMonitor DiskHealthMonitor // tmc is used to run an RPC against other vttablets. tmc tmclient.TabletManagerClient @@ -379,7 +372,6 @@ func (tm *TabletManager) Start(tablet *topodatapb.Tablet, config *tabletenv.Tabl tm.tmc = tmclient.NewTabletManagerClient() tm.tmState = newTMState(tm, tablet) tm.actionSema = semaphore.NewWeighted(1) - tm.dhMonitor = newDiskHealthMonitor(tm.BatchCtx) tm._waitForGrantsComplete = make(chan struct{}) tm.baseTabletType = tablet.Type diff --git a/go/vt/vttablet/tabletserver/controller.go b/go/vt/vttablet/tabletserver/controller.go index c4a4bef99fc..ab2875ae27b 100644 --- a/go/vt/vttablet/tabletserver/controller.go +++ b/go/vt/vttablet/tabletserver/controller.go @@ -122,6 +122,9 @@ type Controller interface { // SetDemotePrimaryStalled marks that demote primary is stalled in the state manager. SetDemotePrimaryStalled() + + // IsDiskStalled returns if the disk is stalled. + IsDiskStalled() bool } // Ensure TabletServer satisfies Controller interface. diff --git a/go/vt/vttablet/tabletmanager/disk_health_monitor.go b/go/vt/vttablet/tabletserver/disk_health_monitor.go similarity index 67% rename from go/vt/vttablet/tabletmanager/disk_health_monitor.go rename to go/vt/vttablet/tabletserver/disk_health_monitor.go index e35bc662a12..f477f7fd30c 100644 --- a/go/vt/vttablet/tabletmanager/disk_health_monitor.go +++ b/go/vt/vttablet/tabletserver/disk_health_monitor.go @@ -1,4 +1,20 @@ -package tabletmanager +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletserver import ( "context" @@ -7,8 +23,29 @@ import ( "strconv" "sync" "time" + + "github.com/spf13/pflag" + + "vitess.io/vitess/go/vt/servenv" ) +var ( + stalledDiskWriteDir = "" + stalledDiskWriteTimeout = 30 * time.Second + stalledDiskWriteInterval = 5 * time.Second +) + +func init() { + servenv.OnParseFor("vtcombo", registerInitFlags) + servenv.OnParseFor("vttablet", registerInitFlags) +} + +func registerInitFlags(fs *pflag.FlagSet) { + fs.StringVar(&stalledDiskWriteDir, "disk-write-dir", stalledDiskWriteDir, "if provided, tablet will attempt to write a file to this directory to check if the disk is stalled") + fs.DurationVar(&stalledDiskWriteTimeout, "disk-write-timeout", stalledDiskWriteTimeout, "if writes exceed this duration, the disk is considered stalled") + fs.DurationVar(&stalledDiskWriteInterval, "disk-write-interval", stalledDiskWriteInterval, "how often to write to the disk to check whether it is stalled") +} + type DiskHealthMonitor interface { // IsDiskStalled returns true if the disk is stalled or rejecting writes. IsDiskStalled() bool diff --git a/go/vt/vttablet/tabletmanager/disk_health_monitor_test.go b/go/vt/vttablet/tabletserver/disk_health_monitor_test.go similarity index 85% rename from go/vt/vttablet/tabletmanager/disk_health_monitor_test.go rename to go/vt/vttablet/tabletserver/disk_health_monitor_test.go index 68930f3061d..8b47e40ee79 100644 --- a/go/vt/vttablet/tabletmanager/disk_health_monitor_test.go +++ b/go/vt/vttablet/tabletserver/disk_health_monitor_test.go @@ -1,4 +1,20 @@ -package tabletmanager +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletserver import ( "context" diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index 4512b26f177..0ccd0e42735 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -97,6 +97,7 @@ type stateManager struct { replHealthy bool demotePrimaryStalled bool lameduck bool + diskHealthMonitor DiskHealthMonitor alsoAllow []topodatapb.TabletType reason string transitionErr error @@ -777,7 +778,7 @@ func (sm *stateManager) IsServing() bool { } func (sm *stateManager) isServingLocked() bool { - return sm.state == StateServing && sm.wantState == StateServing && sm.replHealthy && !sm.demotePrimaryStalled && !sm.lameduck + return sm.state == StateServing && sm.wantState == StateServing && sm.replHealthy && !sm.demotePrimaryStalled && !sm.lameduck && !sm.diskHealthMonitor.IsDiskStalled() } func (sm *stateManager) AppendDetails(details []*kv) []*kv { diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index f8059d6edea..99a3e7e681d 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -41,7 +41,9 @@ import ( var testNow = time.Now() func TestStateManagerStateByName(t *testing.T) { - sm := &stateManager{} + sm := &stateManager{ + diskHealthMonitor: newNoopDiskHealthMonitor(), + } sm.replHealthy = true sm.wantState = StateServing @@ -147,6 +149,29 @@ func TestStateManagerUnservePrimary(t *testing.T) { assert.Equal(t, StateNotServing, sm.state) } +type testDiskMonitor struct { + isDiskStalled bool +} + +func (t *testDiskMonitor) IsDiskStalled() bool { + return t.isDiskStalled +} + +// TestIsServingLocked tests isServingLocked() functionality. +func TestIsServingLocked(t *testing.T) { + sm := newTestStateManager() + defer sm.StopService() + tdm := &testDiskMonitor{isDiskStalled: false} + sm.diskHealthMonitor = tdm + + err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, "") + require.NoError(t, err) + require.True(t, sm.isServingLocked()) + + tdm.isDiskStalled = true + require.False(t, sm.isServingLocked()) +} + func TestStateManagerUnserveNonPrimary(t *testing.T) { sm := newTestStateManager() defer sm.StopService() @@ -778,23 +803,24 @@ func newTestStateManager() *stateManager { parser := sqlparser.NewTestParser() env := tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "StateManagerTest") sm := &stateManager{ - statelessql: NewQueryList("stateless", parser), - statefulql: NewQueryList("stateful", parser), - olapql: NewQueryList("olap", parser), - hs: newHealthStreamer(env, &topodatapb.TabletAlias{}, schema.NewEngine(env)), - se: &testSchemaEngine{}, - rt: &testReplTracker{lag: 1 * time.Second}, - vstreamer: &testSubcomponent{}, - tracker: &testSubcomponent{}, - watcher: &testSubcomponent{}, - qe: &testQueryEngine{}, - txThrottler: &testTxThrottler{}, - te: &testTxEngine{}, - messager: &testSubcomponent{}, - ddle: &testOnlineDDLExecutor{}, - throttler: &testLagThrottler{}, - tableGC: &testTableGC{}, - rw: newRequestsWaiter(), + statelessql: NewQueryList("stateless", parser), + statefulql: NewQueryList("stateful", parser), + olapql: NewQueryList("olap", parser), + hs: newHealthStreamer(env, &topodatapb.TabletAlias{}, schema.NewEngine(env)), + se: &testSchemaEngine{}, + rt: &testReplTracker{lag: 1 * time.Second}, + vstreamer: &testSubcomponent{}, + tracker: &testSubcomponent{}, + watcher: &testSubcomponent{}, + qe: &testQueryEngine{}, + txThrottler: &testTxThrottler{}, + te: &testTxEngine{}, + messager: &testSubcomponent{}, + ddle: &testOnlineDDLExecutor{}, + diskHealthMonitor: newNoopDiskHealthMonitor(), + throttler: &testLagThrottler{}, + tableGC: &testTableGC{}, + rw: newRequestsWaiter(), } sm.Init(env, &querypb.Target{}) sm.hs.InitDBConfig(&querypb.Target{}) diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 30f73d2d818..b65b3949354 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -190,23 +190,24 @@ func NewTabletServer(ctx context.Context, env *vtenv.Environment, name string, c tsv.onlineDDLExecutor = onlineddl.NewExecutor(tsv, alias, topoServer, tsv.lagThrottler, tabletTypeFunc, tsv.onlineDDLExecutorToggleTableBuffer, tsv.tableGC.RequestChecks, tsv.te.preparedPool.IsEmptyForTable) tsv.sm = &stateManager{ - statelessql: tsv.statelessql, - statefulql: tsv.statefulql, - olapql: tsv.olapql, - hs: tsv.hs, - se: tsv.se, - rt: tsv.rt, - vstreamer: tsv.vstreamer, - tracker: tsv.tracker, - watcher: tsv.watcher, - qe: tsv.qe, - txThrottler: tsv.txThrottler, - te: tsv.te, - messager: tsv.messager, - ddle: tsv.onlineDDLExecutor, - throttler: tsv.lagThrottler, - tableGC: tsv.tableGC, - rw: newRequestsWaiter(), + statelessql: tsv.statelessql, + statefulql: tsv.statefulql, + olapql: tsv.olapql, + hs: tsv.hs, + se: tsv.se, + rt: tsv.rt, + vstreamer: tsv.vstreamer, + tracker: tsv.tracker, + watcher: tsv.watcher, + qe: tsv.qe, + txThrottler: tsv.txThrottler, + te: tsv.te, + messager: tsv.messager, + ddle: tsv.onlineDDLExecutor, + throttler: tsv.lagThrottler, + tableGC: tsv.tableGC, + rw: newRequestsWaiter(), + diskHealthMonitor: newDiskHealthMonitor(ctx), } tsv.exporter.NewGaugeFunc("TabletState", "Tablet server state", func() int64 { return int64(tsv.sm.State()) }) @@ -767,6 +768,11 @@ func (tsv *TabletServer) SetDemotePrimaryStalled() { tsv.BroadcastHealth() } +// IsDiskStalled returns if the disk is stalled or not. +func (tsv *TabletServer) IsDiskStalled() bool { + return tsv.sm.diskHealthMonitor.IsDiskStalled() +} + // CreateTransaction creates the metadata for a 2PC transaction. func (tsv *TabletServer) CreateTransaction(ctx context.Context, target *querypb.Target, dtid string, participants []*querypb.Target) (err error) { return tsv.execRequest( diff --git a/go/vt/vttablet/tabletservermock/controller.go b/go/vt/vttablet/tabletservermock/controller.go index a5242751454..21b38755302 100644 --- a/go/vt/vttablet/tabletservermock/controller.go +++ b/go/vt/vttablet/tabletservermock/controller.go @@ -279,6 +279,12 @@ func (tqsc *Controller) SetDemotePrimaryStalled() { tqsc.MethodCalled["SetDemotePrimaryStalled"] = true } +// IsDiskStalled is part of the tabletserver.Controller interface +func (tqsc *Controller) IsDiskStalled() bool { + tqsc.MethodCalled["IsDiskStalled"] = true + return false +} + // EnterLameduck implements tabletserver.Controller. func (tqsc *Controller) EnterLameduck() { tqsc.mu.Lock() diff --git a/proto/replicationdata.proto b/proto/replicationdata.proto index e788fc64bc3..eba4d323ee6 100644 --- a/proto/replicationdata.proto +++ b/proto/replicationdata.proto @@ -105,4 +105,5 @@ message FullStatus { uint32 semi_sync_wait_for_replica_count = 20; bool super_read_only = 21; replicationdata.Configuration replication_configuration = 22; + bool disk_stalled = 23; } diff --git a/web/vtadmin/src/proto/vtadmin.d.ts b/web/vtadmin/src/proto/vtadmin.d.ts index 4411c436083..5a8859b90e7 100644 --- a/web/vtadmin/src/proto/vtadmin.d.ts +++ b/web/vtadmin/src/proto/vtadmin.d.ts @@ -48595,6 +48595,9 @@ export namespace replicationdata { /** FullStatus replication_configuration */ replication_configuration?: (replicationdata.IConfiguration|null); + + /** FullStatus disk_stalled */ + disk_stalled?: (boolean|null); } /** Represents a FullStatus. */ @@ -48672,6 +48675,9 @@ export namespace replicationdata { /** FullStatus replication_configuration. */ public replication_configuration?: (replicationdata.IConfiguration|null); + /** FullStatus disk_stalled. */ + public disk_stalled: boolean; + /** * Creates a new FullStatus instance using the specified properties. * @param [properties] Properties to set diff --git a/web/vtadmin/src/proto/vtadmin.js b/web/vtadmin/src/proto/vtadmin.js index 457ae6e4214..cafe51c0058 100644 --- a/web/vtadmin/src/proto/vtadmin.js +++ b/web/vtadmin/src/proto/vtadmin.js @@ -118108,6 +118108,7 @@ export const replicationdata = $root.replicationdata = (() => { * @property {number|null} [semi_sync_wait_for_replica_count] FullStatus semi_sync_wait_for_replica_count * @property {boolean|null} [super_read_only] FullStatus super_read_only * @property {replicationdata.IConfiguration|null} [replication_configuration] FullStatus replication_configuration + * @property {boolean|null} [disk_stalled] FullStatus disk_stalled */ /** @@ -118301,6 +118302,14 @@ export const replicationdata = $root.replicationdata = (() => { */ FullStatus.prototype.replication_configuration = null; + /** + * FullStatus disk_stalled. + * @member {boolean} disk_stalled + * @memberof replicationdata.FullStatus + * @instance + */ + FullStatus.prototype.disk_stalled = false; + /** * Creates a new FullStatus instance using the specified properties. * @function create @@ -118369,6 +118378,8 @@ export const replicationdata = $root.replicationdata = (() => { writer.uint32(/* id 21, wireType 0 =*/168).bool(message.super_read_only); if (message.replication_configuration != null && Object.hasOwnProperty.call(message, "replication_configuration")) $root.replicationdata.Configuration.encode(message.replication_configuration, writer.uint32(/* id 22, wireType 2 =*/178).fork()).ldelim(); + if (message.disk_stalled != null && Object.hasOwnProperty.call(message, "disk_stalled")) + writer.uint32(/* id 23, wireType 0 =*/184).bool(message.disk_stalled); return writer; }; @@ -118491,6 +118502,10 @@ export const replicationdata = $root.replicationdata = (() => { message.replication_configuration = $root.replicationdata.Configuration.decode(reader, reader.uint32()); break; } + case 23: { + message.disk_stalled = reader.bool(); + break; + } default: reader.skipType(tag & 7); break; @@ -118598,6 +118613,9 @@ export const replicationdata = $root.replicationdata = (() => { if (error) return "replication_configuration." + error; } + if (message.disk_stalled != null && message.hasOwnProperty("disk_stalled")) + if (typeof message.disk_stalled !== "boolean") + return "disk_stalled: boolean expected"; return null; }; @@ -118673,6 +118691,8 @@ export const replicationdata = $root.replicationdata = (() => { throw TypeError(".replicationdata.FullStatus.replication_configuration: object expected"); message.replication_configuration = $root.replicationdata.Configuration.fromObject(object.replication_configuration); } + if (object.disk_stalled != null) + message.disk_stalled = Boolean(object.disk_stalled); return message; }; @@ -118716,6 +118736,7 @@ export const replicationdata = $root.replicationdata = (() => { object.semi_sync_wait_for_replica_count = 0; object.super_read_only = false; object.replication_configuration = null; + object.disk_stalled = false; } if (message.server_id != null && message.hasOwnProperty("server_id")) object.server_id = message.server_id; @@ -118764,6 +118785,8 @@ export const replicationdata = $root.replicationdata = (() => { object.super_read_only = message.super_read_only; if (message.replication_configuration != null && message.hasOwnProperty("replication_configuration")) object.replication_configuration = $root.replicationdata.Configuration.toObject(message.replication_configuration, options); + if (message.disk_stalled != null && message.hasOwnProperty("disk_stalled")) + object.disk_stalled = message.disk_stalled; return object; }; From 19890cfb8ef7bd486e475b58a713d89fc2c29282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Taylor?= Date: Tue, 28 Jan 2025 11:09:53 +0100 Subject: [PATCH 05/68] Optimise AST rewriting (#17623) Signed-off-by: Andres Taylor Signed-off-by: Harshit Gangal Co-authored-by: Harshit Gangal --- go/vt/sqlparser/ast_rewriting.go | 557 -------------- go/vt/sqlparser/ast_rewriting_test.go | 565 -------------- go/vt/sqlparser/bind_var_needs.go | 18 +- go/vt/sqlparser/normalizer.go | 786 ++++++++++++++++---- go/vt/sqlparser/normalizer_test.go | 612 ++++++++++++++- go/vt/sqlparser/redact_query.go | 4 +- go/vt/sqlparser/utils.go | 5 +- go/vt/vtgate/executor_test.go | 24 + go/vt/vtgate/planbuilder/builder.go | 6 +- go/vt/vtgate/planbuilder/simplifier_test.go | 12 +- go/vt/vtgate/semantics/typer_test.go | 17 +- 11 files changed, 1279 insertions(+), 1327 deletions(-) delete mode 100644 go/vt/sqlparser/ast_rewriting.go delete mode 100644 go/vt/sqlparser/ast_rewriting_test.go diff --git a/go/vt/sqlparser/ast_rewriting.go b/go/vt/sqlparser/ast_rewriting.go deleted file mode 100644 index 05e7e290fc1..00000000000 --- a/go/vt/sqlparser/ast_rewriting.go +++ /dev/null @@ -1,557 +0,0 @@ -/* -Copyright 2020 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sqlparser - -import ( - "strconv" - "strings" - - querypb "vitess.io/vitess/go/vt/proto/query" - vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" - "vitess.io/vitess/go/vt/sysvars" - "vitess.io/vitess/go/vt/vterrors" -) - -var HasValueSubQueryBaseName = []byte("__sq_has_values") - -// SQLSelectLimitUnset default value for sql_select_limit not set. -const SQLSelectLimitUnset = -1 - -// RewriteASTResult contains the rewritten ast and meta information about it -type RewriteASTResult struct { - *BindVarNeeds - AST Statement // The rewritten AST -} - -type VSchemaViews interface { - FindView(name TableName) TableStatement -} - -// PrepareAST will normalize the query -func PrepareAST( - in Statement, - reservedVars *ReservedVars, - bindVars map[string]*querypb.BindVariable, - parameterize bool, - keyspace string, - selectLimit int, - setVarComment string, - sysVars map[string]string, - fkChecksState *bool, - views VSchemaViews, -) (*RewriteASTResult, error) { - if parameterize { - err := Normalize(in, reservedVars, bindVars) - if err != nil { - return nil, err - } - } - return RewriteAST(in, keyspace, selectLimit, setVarComment, sysVars, fkChecksState, views) -} - -// RewriteAST rewrites the whole AST, replacing function calls and adding column aliases to queries. -// SET_VAR comments are also added to the AST if required. -func RewriteAST( - in Statement, - keyspace string, - selectLimit int, - setVarComment string, - sysVars map[string]string, - fkChecksState *bool, - views VSchemaViews, -) (*RewriteASTResult, error) { - er := newASTRewriter(keyspace, selectLimit, setVarComment, sysVars, fkChecksState, views) - er.shouldRewriteDatabaseFunc = shouldRewriteDatabaseFunc(in) - result := SafeRewrite(in, er.rewriteDown, er.rewriteUp) - if er.err != nil { - return nil, er.err - } - - out, ok := result.(Statement) - if !ok { - return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "statement rewriting returned a non statement: %s", String(out)) - } - - r := &RewriteASTResult{ - AST: out, - BindVarNeeds: er.bindVars, - } - return r, nil -} - -func shouldRewriteDatabaseFunc(in Statement) bool { - selct, ok := in.(*Select) - if !ok { - return false - } - if len(selct.From) != 1 { - return false - } - aliasedTable, ok := selct.From[0].(*AliasedTableExpr) - if !ok { - return false - } - tableName, ok := aliasedTable.Expr.(TableName) - if !ok { - return false - } - return tableName.Name.String() == "dual" -} - -type astRewriter struct { - bindVars *BindVarNeeds - shouldRewriteDatabaseFunc bool - err error - - // we need to know this to make a decision if we can safely rewrite JOIN USING => JOIN ON - hasStarInSelect bool - - keyspace string - selectLimit int - setVarComment string - fkChecksState *bool - sysVars map[string]string - views VSchemaViews -} - -func newASTRewriter(keyspace string, selectLimit int, setVarComment string, sysVars map[string]string, fkChecksState *bool, views VSchemaViews) *astRewriter { - return &astRewriter{ - bindVars: &BindVarNeeds{}, - keyspace: keyspace, - selectLimit: selectLimit, - setVarComment: setVarComment, - fkChecksState: fkChecksState, - sysVars: sysVars, - views: views, - } -} - -const ( - // LastInsertIDName is a reserved bind var name for last_insert_id() - LastInsertIDName = "__lastInsertId" - - // DBVarName is a reserved bind var name for database() - DBVarName = "__vtdbname" - - // FoundRowsName is a reserved bind var name for found_rows() - FoundRowsName = "__vtfrows" - - // RowCountName is a reserved bind var name for row_count() - RowCountName = "__vtrcount" - - // UserDefinedVariableName is what we prepend bind var names for user defined variables - UserDefinedVariableName = "__vtudv" -) - -func (er *astRewriter) rewriteAliasedExpr(node *AliasedExpr) (*BindVarNeeds, error) { - inner := newASTRewriter(er.keyspace, er.selectLimit, er.setVarComment, er.sysVars, nil, er.views) - inner.shouldRewriteDatabaseFunc = er.shouldRewriteDatabaseFunc - tmp := SafeRewrite(node.Expr, inner.rewriteDown, inner.rewriteUp) - newExpr, ok := tmp.(Expr) - if !ok { - return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "failed to rewrite AST. function expected to return Expr returned a %s", String(tmp)) - } - node.Expr = newExpr - return inner.bindVars, nil -} - -func (er *astRewriter) rewriteDown(node SQLNode, _ SQLNode) bool { - switch node := node.(type) { - case *Select: - er.visitSelect(node) - case *PrepareStmt, *ExecuteStmt: - return false // nothing to rewrite here. - } - return true -} - -func (er *astRewriter) rewriteUp(cursor *Cursor) bool { - // Add SET_VAR comment to this node if it supports it and is needed - if supportOptimizerHint, supportsOptimizerHint := cursor.Node().(SupportOptimizerHint); supportsOptimizerHint { - if er.setVarComment != "" { - newComments, err := supportOptimizerHint.GetParsedComments().AddQueryHint(er.setVarComment) - if err != nil { - er.err = err - return false - } - supportOptimizerHint.SetComments(newComments) - } - if er.fkChecksState != nil { - newComments := supportOptimizerHint.GetParsedComments().SetMySQLSetVarValue(sysvars.ForeignKeyChecks, FkChecksStateString(er.fkChecksState)) - supportOptimizerHint.SetComments(newComments) - } - } - - switch node := cursor.Node().(type) { - case *Union: - er.rewriteUnion(node) - case *FuncExpr: - er.funcRewrite(cursor, node) - case *Variable: - er.rewriteVariable(cursor, node) - case *Subquery: - er.unnestSubQueries(cursor, node) - case *NotExpr: - er.rewriteNotExpr(cursor, node) - case *AliasedTableExpr: - er.rewriteAliasedTable(cursor, node) - case *ShowBasic: - er.rewriteShowBasic(node) - case *ExistsExpr: - er.existsRewrite(cursor, node) - case DistinctableAggr: - er.rewriteDistinctableAggr(cursor, node) - } - return true -} - -func (er *astRewriter) rewriteUnion(node *Union) { - // set select limit if explicitly not set when sql_select_limit is set on the connection. - if er.selectLimit > 0 && node.Limit == nil { - node.Limit = &Limit{Rowcount: NewIntLiteral(strconv.Itoa(er.selectLimit))} - } -} - -func (er *astRewriter) rewriteAliasedTable(cursor *Cursor, node *AliasedTableExpr) { - aliasTableName, ok := node.Expr.(TableName) - if !ok { - return - } - - // Qualifier should not be added to dual table - tblName := aliasTableName.Name.String() - if tblName == "dual" { - return - } - - if SystemSchema(er.keyspace) { - if aliasTableName.Qualifier.IsEmpty() { - aliasTableName.Qualifier = NewIdentifierCS(er.keyspace) - node.Expr = aliasTableName - cursor.Replace(node) - } - return - } - - // Could we be dealing with a view? - if er.views == nil { - return - } - view := er.views.FindView(aliasTableName) - if view == nil { - return - } - - // Aha! It's a view. Let's replace it with a derived table - node.Expr = &DerivedTable{Select: Clone(view)} // TODO: this is a bit hacky. We want to update the schema def so it contains new types - if node.As.IsEmpty() { - node.As = NewIdentifierCS(tblName) - } -} - -func (er *astRewriter) rewriteShowBasic(node *ShowBasic) { - if node.Command == VariableGlobal || node.Command == VariableSession { - varsToAdd := sysvars.GetInterestingVariables() - for _, sysVar := range varsToAdd { - er.bindVars.AddSysVar(sysVar) - } - } -} - -func (er *astRewriter) rewriteNotExpr(cursor *Cursor, node *NotExpr) { - switch inner := node.Expr.(type) { - case *ComparisonExpr: - // not col = 42 => col != 42 - // not col > 42 => col <= 42 - // etc - canChange, inverse := inverseOp(inner.Operator) - if canChange { - inner.Operator = inverse - cursor.Replace(inner) - } - case *NotExpr: - // not not true => true - cursor.Replace(inner.Expr) - case BoolVal: - // not true => false - inner = !inner - cursor.Replace(inner) - } -} - -func (er *astRewriter) rewriteVariable(cursor *Cursor, node *Variable) { - // Iff we are in SET, we want to change the scope of variables if a modifier has been set - // and only on the lhs of the assignment: - // set session sql_mode = @someElse - // here we need to change the scope of `sql_mode` and not of `@someElse` - if v, isSet := cursor.Parent().(*SetExpr); isSet && v.Var == node { - return - } - // no rewriting for global scope variable. - // this should be returned from the underlying database. - switch node.Scope { - case VariableScope: - er.udvRewrite(cursor, node) - case SessionScope, NextTxScope: - er.sysVarRewrite(cursor, node) - } -} - -func (er *astRewriter) visitSelect(node *Select) { - for _, col := range node.SelectExprs { - if _, hasStar := col.(*StarExpr); hasStar { - er.hasStarInSelect = true - continue - } - - aliasedExpr, ok := col.(*AliasedExpr) - if !ok || aliasedExpr.As.NotEmpty() { - continue - } - buf := NewTrackedBuffer(nil) - aliasedExpr.Expr.Format(buf) - // select last_insert_id() -> select :__lastInsertId as `last_insert_id()` - innerBindVarNeeds, err := er.rewriteAliasedExpr(aliasedExpr) - if err != nil { - er.err = err - return - } - if innerBindVarNeeds.HasRewrites() { - aliasedExpr.As = NewIdentifierCI(buf.String()) - } - er.bindVars.MergeWith(innerBindVarNeeds) - - } - // set select limit if explicitly not set when sql_select_limit is set on the connection. - if er.selectLimit > 0 && node.Limit == nil { - node.Limit = &Limit{Rowcount: NewIntLiteral(strconv.Itoa(er.selectLimit))} - } -} - -func inverseOp(i ComparisonExprOperator) (bool, ComparisonExprOperator) { - switch i { - case EqualOp: - return true, NotEqualOp - case LessThanOp: - return true, GreaterEqualOp - case GreaterThanOp: - return true, LessEqualOp - case LessEqualOp: - return true, GreaterThanOp - case GreaterEqualOp: - return true, LessThanOp - case NotEqualOp: - return true, EqualOp - case InOp: - return true, NotInOp - case NotInOp: - return true, InOp - case LikeOp: - return true, NotLikeOp - case NotLikeOp: - return true, LikeOp - case RegexpOp: - return true, NotRegexpOp - case NotRegexpOp: - return true, RegexpOp - } - - return false, i -} - -func (er *astRewriter) sysVarRewrite(cursor *Cursor, node *Variable) { - lowered := node.Name.Lowered() - - var found bool - if er.sysVars != nil { - _, found = er.sysVars[lowered] - } - - switch lowered { - case sysvars.Autocommit.Name, - sysvars.Charset.Name, - sysvars.ClientFoundRows.Name, - sysvars.DDLStrategy.Name, - sysvars.MigrationContext.Name, - sysvars.Names.Name, - sysvars.TransactionMode.Name, - sysvars.ReadAfterWriteGTID.Name, - sysvars.ReadAfterWriteTimeOut.Name, - sysvars.SessionEnableSystemSettings.Name, - sysvars.SessionTrackGTIDs.Name, - sysvars.SessionUUID.Name, - sysvars.SkipQueryPlanCache.Name, - sysvars.Socket.Name, - sysvars.SQLSelectLimit.Name, - sysvars.Version.Name, - sysvars.VersionComment.Name, - sysvars.QueryTimeout.Name, - sysvars.Workload.Name: - found = true - } - - if found { - cursor.Replace(bindVarExpression("__vt" + lowered)) - er.bindVars.AddSysVar(lowered) - } -} - -func (er *astRewriter) udvRewrite(cursor *Cursor, node *Variable) { - udv := strings.ToLower(node.Name.CompliantName()) - cursor.Replace(bindVarExpression(UserDefinedVariableName + udv)) - er.bindVars.AddUserDefVar(udv) -} - -var funcRewrites = map[string]string{ - "last_insert_id": LastInsertIDName, - "database": DBVarName, - "schema": DBVarName, - "found_rows": FoundRowsName, - "row_count": RowCountName, -} - -func (er *astRewriter) funcRewrite(cursor *Cursor, node *FuncExpr) { - lowered := node.Name.Lowered() - if lowered == "last_insert_id" && len(node.Exprs) > 0 { - // if we are dealing with is LAST_INSERT_ID() with an argument, we don't need to rewrite it. - // with an argument, this is an identity function that will update the session state and - // sets the correct fields in the OK TCP packet that we send back - return - } - bindVar, found := funcRewrites[lowered] - if !found || (bindVar == DBVarName && !er.shouldRewriteDatabaseFunc) { - return - } - if len(node.Exprs) > 0 { - er.err = vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "Argument to %s() not supported", lowered) - return - } - cursor.Replace(bindVarExpression(bindVar)) - er.bindVars.AddFuncResult(bindVar) -} - -func (er *astRewriter) unnestSubQueries(cursor *Cursor, subquery *Subquery) { - if _, isExists := cursor.Parent().(*ExistsExpr); isExists { - return - } - sel, isSimpleSelect := subquery.Select.(*Select) - if !isSimpleSelect { - return - } - - if len(sel.SelectExprs) != 1 || - len(sel.OrderBy) != 0 || - sel.GroupBy != nil || - len(sel.From) != 1 || - sel.Where != nil || - sel.Having != nil || - sel.Limit != nil || sel.Lock != NoLock { - return - } - - aliasedTable, ok := sel.From[0].(*AliasedTableExpr) - if !ok { - return - } - table, ok := aliasedTable.Expr.(TableName) - if !ok || table.Name.String() != "dual" { - return - } - expr, ok := sel.SelectExprs[0].(*AliasedExpr) - if !ok { - return - } - _, isColName := expr.Expr.(*ColName) - if isColName { - // If we find a single col-name in a `dual` subquery, we can be pretty sure the user is returning a column - // already projected. - // `select 1 as x, (select x)` - // is perfectly valid - any aliased columns to the left are available inside subquery scopes - return - } - er.bindVars.NoteRewrite() - // we need to make sure that the inner expression also gets rewritten, - // so we fire off another rewriter traversal here - rewritten := SafeRewrite(expr.Expr, er.rewriteDown, er.rewriteUp) - - // Here we need to handle the subquery rewrite in case in occurs in an IN clause - // For example, SELECT id FROM user WHERE id IN (SELECT 1 FROM DUAL) - // Here we cannot rewrite the query to SELECT id FROM user WHERE id IN 1, since that is syntactically wrong - // We must rewrite it to SELECT id FROM user WHERE id IN (1) - // Find more cases in the test file - rewrittenExpr, isExpr := rewritten.(Expr) - _, isColTuple := rewritten.(ColTuple) - comparisonExpr, isCompExpr := cursor.Parent().(*ComparisonExpr) - // Check that the parent is a comparison operator with IN or NOT IN operation. - // Also, if rewritten is already a ColTuple (like a subquery), then we do not need this - // We also need to check that rewritten is an Expr, if it is then we can rewrite it as a ValTuple - if isCompExpr && (comparisonExpr.Operator == InOp || comparisonExpr.Operator == NotInOp) && !isColTuple && isExpr { - cursor.Replace(ValTuple{rewrittenExpr}) - return - } - - cursor.Replace(rewritten) -} - -func (er *astRewriter) existsRewrite(cursor *Cursor, node *ExistsExpr) { - sel, ok := node.Subquery.Select.(*Select) - if !ok { - return - } - - if sel.Having != nil { - // If the query has HAVING, we can't take any shortcuts - return - } - - if sel.GroupBy == nil && sel.SelectExprs.AllAggregation() { - // in these situations, we are guaranteed to always get a non-empty result, - // so we can replace the EXISTS with a literal true - cursor.Replace(BoolVal(true)) - } - - // If we are not doing HAVING, we can safely replace all select expressions with a - // single `1` and remove any grouping - sel.SelectExprs = SelectExprs{ - &AliasedExpr{Expr: NewIntLiteral("1")}, - } - sel.GroupBy = nil -} - -// rewriteDistinctableAggr removed Distinct from Max and Min Aggregations as it does not impact the result. But, makes the plan simpler. -func (er *astRewriter) rewriteDistinctableAggr(cursor *Cursor, node DistinctableAggr) { - if !node.IsDistinct() { - return - } - switch aggr := node.(type) { - case *Max, *Min: - aggr.SetDistinct(false) - er.bindVars.NoteRewrite() - } -} - -func bindVarExpression(name string) Expr { - return NewArgument(name) -} - -// SystemSchema returns true if the schema passed is system schema -func SystemSchema(schema string) bool { - return strings.EqualFold(schema, "information_schema") || - strings.EqualFold(schema, "performance_schema") || - strings.EqualFold(schema, "sys") || - strings.EqualFold(schema, "mysql") -} diff --git a/go/vt/sqlparser/ast_rewriting_test.go b/go/vt/sqlparser/ast_rewriting_test.go deleted file mode 100644 index 8b3e3d44c54..00000000000 --- a/go/vt/sqlparser/ast_rewriting_test.go +++ /dev/null @@ -1,565 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sqlparser - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - - "vitess.io/vitess/go/vt/sysvars" - - "github.com/stretchr/testify/require" -) - -type testCaseSetVar struct { - in, expected, setVarComment string -} - -type testCaseSysVar struct { - in, expected string - sysVar map[string]string -} - -type myTestCase struct { - in, expected string - liid, db, foundRows, rowCount, rawGTID, rawTimeout, sessTrackGTID bool - ddlStrategy, migrationContext, sessionUUID, sessionEnableSystemSettings bool - udv int - autocommit, foreignKeyChecks, clientFoundRows, skipQueryPlanCache, socket, queryTimeout bool - sqlSelectLimit, transactionMode, workload, version, versionComment bool -} - -func TestRewrites(in *testing.T) { - tests := []myTestCase{{ - in: "SELECT 42", - expected: "SELECT 42", - // no bindvar needs - }, { - in: "SELECT @@version", - expected: "SELECT :__vtversion as `@@version`", - version: true, - }, { - in: "SELECT @@query_timeout", - expected: "SELECT :__vtquery_timeout as `@@query_timeout`", - queryTimeout: true, - }, { - in: "SELECT @@version_comment", - expected: "SELECT :__vtversion_comment as `@@version_comment`", - versionComment: true, - }, { - in: "SELECT @@enable_system_settings", - expected: "SELECT :__vtenable_system_settings as `@@enable_system_settings`", - sessionEnableSystemSettings: true, - }, { - in: "SELECT last_insert_id()", - expected: "SELECT :__lastInsertId as `last_insert_id()`", - liid: true, - }, { - in: "SELECT database()", - expected: "SELECT :__vtdbname as `database()`", - db: true, - }, { - in: "SELECT database() from test", - expected: "SELECT database() from test", - // no bindvar needs - }, { - in: "SELECT last_insert_id() as test", - expected: "SELECT :__lastInsertId as test", - liid: true, - }, { - in: "SELECT last_insert_id() + database()", - expected: "SELECT :__lastInsertId + :__vtdbname as `last_insert_id() + database()`", - db: true, liid: true, - }, { - // unnest database() call - in: "select (select database()) from test", - expected: "select database() as `(select database() from dual)` from test", - // no bindvar needs - }, { - // unnest database() call - in: "select (select database() from dual) from test", - expected: "select database() as `(select database() from dual)` from test", - // no bindvar needs - }, { - in: "select (select database() from dual) from dual", - expected: "select :__vtdbname as `(select database() from dual)` from dual", - db: true, - }, { - // don't unnest solo columns - in: "select 1 as foobar, (select foobar)", - expected: "select 1 as foobar, (select foobar from dual) from dual", - }, { - in: "select id from user where database()", - expected: "select id from user where database()", - // no bindvar needs - }, { - in: "select table_name from information_schema.tables where table_schema = database()", - expected: "select table_name from information_schema.tables where table_schema = database()", - // no bindvar needs - }, { - in: "select schema()", - expected: "select :__vtdbname as `schema()`", - db: true, - }, { - in: "select found_rows()", - expected: "select :__vtfrows as `found_rows()`", - foundRows: true, - }, { - in: "select @`x y`", - expected: "select :__vtudvx_y as `@``x y``` from dual", - udv: 1, - }, { - in: "select id from t where id = @x and val = @y", - expected: "select id from t where id = :__vtudvx and val = :__vtudvy", - db: false, udv: 2, - }, { - in: "insert into t(id) values(@xyx)", - expected: "insert into t(id) values(:__vtudvxyx)", - db: false, udv: 1, - }, { - in: "select row_count()", - expected: "select :__vtrcount as `row_count()`", - rowCount: true, - }, { - in: "SELECT lower(database())", - expected: "SELECT lower(:__vtdbname) as `lower(database())`", - db: true, - }, { - in: "SELECT @@autocommit", - expected: "SELECT :__vtautocommit as `@@autocommit`", - autocommit: true, - }, { - in: "SELECT @@client_found_rows", - expected: "SELECT :__vtclient_found_rows as `@@client_found_rows`", - clientFoundRows: true, - }, { - in: "SELECT @@skip_query_plan_cache", - expected: "SELECT :__vtskip_query_plan_cache as `@@skip_query_plan_cache`", - skipQueryPlanCache: true, - }, { - in: "SELECT @@sql_select_limit", - expected: "SELECT :__vtsql_select_limit as `@@sql_select_limit`", - sqlSelectLimit: true, - }, { - in: "SELECT @@transaction_mode", - expected: "SELECT :__vttransaction_mode as `@@transaction_mode`", - transactionMode: true, - }, { - in: "SELECT @@workload", - expected: "SELECT :__vtworkload as `@@workload`", - workload: true, - }, { - in: "SELECT @@socket", - expected: "SELECT :__vtsocket as `@@socket`", - socket: true, - }, { - in: "select (select 42) from dual", - expected: "select 42 as `(select 42 from dual)` from dual", - }, { - in: "select * from user where col = (select 42)", - expected: "select * from user where col = 42", - }, { - in: "select * from (select 42) as t", // this is not an expression, and should not be rewritten - expected: "select * from (select 42) as t", - }, { - in: `select (select (select (select (select (select last_insert_id()))))) as x`, - expected: "select :__lastInsertId as x from dual", - liid: true, - }, { - in: `select * from user where col = @@ddl_strategy`, - expected: "select * from user where col = :__vtddl_strategy", - ddlStrategy: true, - }, { - in: `select * from user where col = @@migration_context`, - expected: "select * from user where col = :__vtmigration_context", - migrationContext: true, - }, { - in: `select * from user where col = @@read_after_write_gtid OR col = @@read_after_write_timeout OR col = @@session_track_gtids`, - expected: "select * from user where col = :__vtread_after_write_gtid or col = :__vtread_after_write_timeout or col = :__vtsession_track_gtids", - rawGTID: true, rawTimeout: true, sessTrackGTID: true, - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual)", - expected: "SELECT * FROM tbl WHERE id IN (1)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT last_insert_id() FROM dual)", - expected: "SELECT * FROM tbl WHERE id IN (:__lastInsertId)", - liid: true, - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT (SELECT 1 FROM dual WHERE 1 = 0) FROM dual)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1,2 FROM dual)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1,2 FROM dual)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual ORDER BY 1)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual ORDER BY 1)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT id FROM user GROUP BY id)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT id FROM user GROUP BY id)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual, user)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual, user)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual limit 1)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual limit 1)", - }, { - // SELECT * behaves different depending the join type used, so if that has been used, we won't rewrite - in: "SELECT * FROM A JOIN B USING (id1,id2,id3)", - expected: "SELECT * FROM A JOIN B USING (id1,id2,id3)", - }, { - in: "CALL proc(@foo)", - expected: "CALL proc(:__vtudvfoo)", - udv: 1, - }, { - in: "SELECT * FROM tbl WHERE NOT id = 42", - expected: "SELECT * FROM tbl WHERE id != 42", - }, { - in: "SELECT * FROM tbl WHERE not id < 12", - expected: "SELECT * FROM tbl WHERE id >= 12", - }, { - in: "SELECT * FROM tbl WHERE not id > 12", - expected: "SELECT * FROM tbl WHERE id <= 12", - }, { - in: "SELECT * FROM tbl WHERE not id <= 33", - expected: "SELECT * FROM tbl WHERE id > 33", - }, { - in: "SELECT * FROM tbl WHERE not id >= 33", - expected: "SELECT * FROM tbl WHERE id < 33", - }, { - in: "SELECT * FROM tbl WHERE not id != 33", - expected: "SELECT * FROM tbl WHERE id = 33", - }, { - in: "SELECT * FROM tbl WHERE not id in (1,2,3)", - expected: "SELECT * FROM tbl WHERE id not in (1,2,3)", - }, { - in: "SELECT * FROM tbl WHERE not id not in (1,2,3)", - expected: "SELECT * FROM tbl WHERE id in (1,2,3)", - }, { - in: "SELECT * FROM tbl WHERE not id not in (1,2,3)", - expected: "SELECT * FROM tbl WHERE id in (1,2,3)", - }, { - in: "SELECT * FROM tbl WHERE not id like '%foobar'", - expected: "SELECT * FROM tbl WHERE id not like '%foobar'", - }, { - in: "SELECT * FROM tbl WHERE not id not like '%foobar'", - expected: "SELECT * FROM tbl WHERE id like '%foobar'", - }, { - in: "SELECT * FROM tbl WHERE not id regexp '%foobar'", - expected: "SELECT * FROM tbl WHERE id not regexp '%foobar'", - }, { - in: "SELECT * FROM tbl WHERE not id not regexp '%foobar'", - expected: "select * from tbl where id regexp '%foobar'", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar)", - expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar limit 100 offset 34)", - expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar limit 100 offset 34)", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2)", - expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar group by col1, col2)", - expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", - }, { - in: "SELECT * FROM tbl WHERE exists(select count(*) from other_table where foo > bar)", - expected: "SELECT * FROM tbl WHERE true", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2 having count(*) > 3)", - expected: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2 having count(*) > 3)", - }, { - in: "SELECT id, name, salary FROM user_details", - expected: "SELECT id, name, salary FROM (select user.id, user.name, user_extra.salary from user join user_extra where user.id = user_extra.user_id) as user_details", - }, { - in: "select max(distinct c1), min(distinct c2), avg(distinct c3), sum(distinct c4), count(distinct c5), group_concat(distinct c6) from tbl", - expected: "select max(c1) as `max(distinct c1)`, min(c2) as `min(distinct c2)`, avg(distinct c3), sum(distinct c4), count(distinct c5), group_concat(distinct c6) from tbl", - }, { - in: "SHOW VARIABLES", - expected: "SHOW VARIABLES", - autocommit: true, - foreignKeyChecks: true, - clientFoundRows: true, - skipQueryPlanCache: true, - sqlSelectLimit: true, - transactionMode: true, - workload: true, - version: true, - versionComment: true, - ddlStrategy: true, - migrationContext: true, - sessionUUID: true, - sessionEnableSystemSettings: true, - rawGTID: true, - rawTimeout: true, - sessTrackGTID: true, - socket: true, - queryTimeout: true, - }, { - in: "SHOW GLOBAL VARIABLES", - expected: "SHOW GLOBAL VARIABLES", - autocommit: true, - foreignKeyChecks: true, - clientFoundRows: true, - skipQueryPlanCache: true, - sqlSelectLimit: true, - transactionMode: true, - workload: true, - version: true, - versionComment: true, - ddlStrategy: true, - migrationContext: true, - sessionUUID: true, - sessionEnableSystemSettings: true, - rawGTID: true, - rawTimeout: true, - sessTrackGTID: true, - socket: true, - queryTimeout: true, - }} - parser := NewTestParser() - for _, tc := range tests { - in.Run(tc.in, func(t *testing.T) { - require := require.New(t) - stmt, err := parser.Parse(tc.in) - require.NoError(err) - - result, err := RewriteAST( - stmt, - "ks", // passing `ks` just to test that no rewriting happens as it is not system schema - SQLSelectLimitUnset, - "", - nil, - nil, - &fakeViews{}, - ) - require.NoError(err) - - expected, err := parser.Parse(tc.expected) - require.NoError(err, "test expectation does not parse [%s]", tc.expected) - - s := String(expected) - assert := assert.New(t) - assert.Equal(s, String(result.AST)) - assert.Equal(tc.liid, result.NeedsFuncResult(LastInsertIDName), "should need last insert id") - assert.Equal(tc.db, result.NeedsFuncResult(DBVarName), "should need database name") - assert.Equal(tc.foundRows, result.NeedsFuncResult(FoundRowsName), "should need found rows") - assert.Equal(tc.rowCount, result.NeedsFuncResult(RowCountName), "should need row count") - assert.Equal(tc.udv, len(result.NeedUserDefinedVariables), "count of user defined variables") - assert.Equal(tc.autocommit, result.NeedsSysVar(sysvars.Autocommit.Name), "should need :__vtautocommit") - assert.Equal(tc.foreignKeyChecks, result.NeedsSysVar(sysvars.ForeignKeyChecks), "should need :__vtforeignKeyChecks") - assert.Equal(tc.clientFoundRows, result.NeedsSysVar(sysvars.ClientFoundRows.Name), "should need :__vtclientFoundRows") - assert.Equal(tc.skipQueryPlanCache, result.NeedsSysVar(sysvars.SkipQueryPlanCache.Name), "should need :__vtskipQueryPlanCache") - assert.Equal(tc.sqlSelectLimit, result.NeedsSysVar(sysvars.SQLSelectLimit.Name), "should need :__vtsqlSelectLimit") - assert.Equal(tc.transactionMode, result.NeedsSysVar(sysvars.TransactionMode.Name), "should need :__vttransactionMode") - assert.Equal(tc.workload, result.NeedsSysVar(sysvars.Workload.Name), "should need :__vtworkload") - assert.Equal(tc.queryTimeout, result.NeedsSysVar(sysvars.QueryTimeout.Name), "should need :__vtquery_timeout") - assert.Equal(tc.ddlStrategy, result.NeedsSysVar(sysvars.DDLStrategy.Name), "should need ddlStrategy") - assert.Equal(tc.migrationContext, result.NeedsSysVar(sysvars.MigrationContext.Name), "should need migrationContext") - assert.Equal(tc.sessionUUID, result.NeedsSysVar(sysvars.SessionUUID.Name), "should need sessionUUID") - assert.Equal(tc.sessionEnableSystemSettings, result.NeedsSysVar(sysvars.SessionEnableSystemSettings.Name), "should need sessionEnableSystemSettings") - assert.Equal(tc.rawGTID, result.NeedsSysVar(sysvars.ReadAfterWriteGTID.Name), "should need rawGTID") - assert.Equal(tc.rawTimeout, result.NeedsSysVar(sysvars.ReadAfterWriteTimeOut.Name), "should need rawTimeout") - assert.Equal(tc.sessTrackGTID, result.NeedsSysVar(sysvars.SessionTrackGTIDs.Name), "should need sessTrackGTID") - assert.Equal(tc.version, result.NeedsSysVar(sysvars.Version.Name), "should need Vitess version") - assert.Equal(tc.versionComment, result.NeedsSysVar(sysvars.VersionComment.Name), "should need Vitess version") - assert.Equal(tc.socket, result.NeedsSysVar(sysvars.Socket.Name), "should need :__vtsocket") - }) - } -} - -type fakeViews struct{} - -func (*fakeViews) FindView(name TableName) TableStatement { - if name.Name.String() != "user_details" { - return nil - } - parser := NewTestParser() - statement, err := parser.Parse("select user.id, user.name, user_extra.salary from user join user_extra where user.id = user_extra.user_id") - if err != nil { - return nil - } - return statement.(TableStatement) -} - -func TestRewritesWithSetVarComment(in *testing.T) { - tests := []testCaseSetVar{{ - in: "select 1", - expected: "select 1", - setVarComment: "", - }, { - in: "select 1", - expected: "select /*+ AA(a) */ 1", - setVarComment: "AA(a)", - }, { - in: "insert /* toto */ into t(id) values(1)", - expected: "insert /*+ AA(a) */ /* toto */ into t(id) values(1)", - setVarComment: "AA(a)", - }, { - in: "select /* toto */ * from t union select * from s", - expected: "select /*+ AA(a) */ /* toto */ * from t union select /*+ AA(a) */ * from s", - setVarComment: "AA(a)", - }, { - in: "vstream /* toto */ * from t1", - expected: "vstream /*+ AA(a) */ /* toto */ * from t1", - setVarComment: "AA(a)", - }, { - in: "stream /* toto */ t from t1", - expected: "stream /*+ AA(a) */ /* toto */ t from t1", - setVarComment: "AA(a)", - }, { - in: "update /* toto */ t set id = 1", - expected: "update /*+ AA(a) */ /* toto */ t set id = 1", - setVarComment: "AA(a)", - }, { - in: "delete /* toto */ from t", - expected: "delete /*+ AA(a) */ /* toto */ from t", - setVarComment: "AA(a)", - }} - - parser := NewTestParser() - for _, tc := range tests { - in.Run(tc.in, func(t *testing.T) { - require := require.New(t) - stmt, err := parser.Parse(tc.in) - require.NoError(err) - - result, err := RewriteAST(stmt, "ks", SQLSelectLimitUnset, tc.setVarComment, nil, nil, &fakeViews{}) - require.NoError(err) - - expected, err := parser.Parse(tc.expected) - require.NoError(err, "test expectation does not parse [%s]", tc.expected) - - assert.Equal(t, String(expected), String(result.AST)) - }) - } -} - -func TestRewritesSysVar(in *testing.T) { - tests := []testCaseSysVar{{ - in: "select @x = @@sql_mode", - expected: "select :__vtudvx = @@sql_mode as `@x = @@sql_mode` from dual", - }, { - in: "select @x = @@sql_mode", - expected: "select :__vtudvx = :__vtsql_mode as `@x = @@sql_mode` from dual", - sysVar: map[string]string{"sql_mode": "' '"}, - }, { - in: "SELECT @@tx_isolation", - expected: "select @@tx_isolation from dual", - }, { - in: "SELECT @@transaction_isolation", - expected: "select @@transaction_isolation from dual", - }, { - in: "SELECT @@session.transaction_isolation", - expected: "select @@session.transaction_isolation from dual", - }, { - in: "SELECT @@tx_isolation", - sysVar: map[string]string{"tx_isolation": "'READ-COMMITTED'"}, - expected: "select :__vttx_isolation as `@@tx_isolation` from dual", - }, { - in: "SELECT @@transaction_isolation", - sysVar: map[string]string{"transaction_isolation": "'READ-COMMITTED'"}, - expected: "select :__vttransaction_isolation as `@@transaction_isolation` from dual", - }, { - in: "SELECT @@session.transaction_isolation", - sysVar: map[string]string{"transaction_isolation": "'READ-COMMITTED'"}, - expected: "select :__vttransaction_isolation as `@@session.transaction_isolation` from dual", - }} - - parser := NewTestParser() - for _, tc := range tests { - in.Run(tc.in, func(t *testing.T) { - require := require.New(t) - stmt, err := parser.Parse(tc.in) - require.NoError(err) - - result, err := RewriteAST(stmt, "ks", SQLSelectLimitUnset, "", tc.sysVar, nil, &fakeViews{}) - require.NoError(err) - - expected, err := parser.Parse(tc.expected) - require.NoError(err, "test expectation does not parse [%s]", tc.expected) - - assert.Equal(t, String(expected), String(result.AST)) - }) - } -} - -func TestRewritesWithDefaultKeyspace(in *testing.T) { - tests := []myTestCase{{ - in: "SELECT 1 from x.test", - expected: "SELECT 1 from x.test", // no change - }, { - in: "SELECT x.col as c from x.test", - expected: "SELECT x.col as c from x.test", // no change - }, { - in: "SELECT 1 from test", - expected: "SELECT 1 from sys.test", - }, { - in: "SELECT 1 from test as t", - expected: "SELECT 1 from sys.test as t", - }, { - in: "SELECT 1 from `test 24` as t", - expected: "SELECT 1 from sys.`test 24` as t", - }, { - in: "SELECT 1, (select 1 from test) from x.y", - expected: "SELECT 1, (select 1 from sys.test) from x.y", - }, { - in: "SELECT 1 from (select 2 from test) t", - expected: "SELECT 1 from (select 2 from sys.test) t", - }, { - in: "SELECT 1 from test where exists(select 2 from test)", - expected: "SELECT 1 from sys.test where exists(select 1 from sys.test)", - }, { - in: "SELECT 1 from dual", - expected: "SELECT 1 from dual", - }, { - in: "SELECT (select 2 from dual) from DUAL", - expected: "SELECT 2 as `(select 2 from dual)` from DUAL", - }} - - parser := NewTestParser() - for _, tc := range tests { - in.Run(tc.in, func(t *testing.T) { - require := require.New(t) - stmt, err := parser.Parse(tc.in) - require.NoError(err) - - result, err := RewriteAST(stmt, "sys", SQLSelectLimitUnset, "", nil, nil, &fakeViews{}) - require.NoError(err) - - expected, err := parser.Parse(tc.expected) - require.NoError(err, "test expectation does not parse [%s]", tc.expected) - - assert.Equal(t, String(expected), String(result.AST)) - }) - } -} - -func TestReservedVars(t *testing.T) { - for _, prefix := range []string{"vtg", "bv"} { - t.Run("prefix_"+prefix, func(t *testing.T) { - reserved := NewReservedVars(prefix, make(BindVars)) - for i := 1; i < 1000; i++ { - require.Equal(t, fmt.Sprintf("%s%d", prefix, i), reserved.nextUnusedVar()) - } - }) - } -} diff --git a/go/vt/sqlparser/bind_var_needs.go b/go/vt/sqlparser/bind_var_needs.go index 1b26919ca03..64e5c528e97 100644 --- a/go/vt/sqlparser/bind_var_needs.go +++ b/go/vt/sqlparser/bind_var_needs.go @@ -22,14 +22,7 @@ type BindVarNeeds struct { NeedSystemVariable, // NeedUserDefinedVariables keeps track of all user defined variables a query is using NeedUserDefinedVariables []string - otherRewrites bool -} - -// MergeWith adds bind vars needs coming from sub scopes -func (bvn *BindVarNeeds) MergeWith(other *BindVarNeeds) { - bvn.NeedFunctionResult = append(bvn.NeedFunctionResult, other.NeedFunctionResult...) - bvn.NeedSystemVariable = append(bvn.NeedSystemVariable, other.NeedSystemVariable...) - bvn.NeedUserDefinedVariables = append(bvn.NeedUserDefinedVariables, other.NeedUserDefinedVariables...) + otherRewrites int } // AddFuncResult adds a function bindvar need @@ -58,14 +51,11 @@ func (bvn *BindVarNeeds) NeedsSysVar(name string) bool { } func (bvn *BindVarNeeds) NoteRewrite() { - bvn.otherRewrites = true + bvn.otherRewrites++ } -func (bvn *BindVarNeeds) HasRewrites() bool { - return bvn.otherRewrites || - len(bvn.NeedFunctionResult) > 0 || - len(bvn.NeedUserDefinedVariables) > 0 || - len(bvn.NeedSystemVariable) > 0 +func (bvn *BindVarNeeds) NumberOfRewrites() int { + return len(bvn.NeedFunctionResult) + len(bvn.NeedUserDefinedVariables) + len(bvn.NeedSystemVariable) + bvn.otherRewrites } func contains(strings []string, name string) bool { diff --git a/go/vt/sqlparser/normalizer.go b/go/vt/sqlparser/normalizer.go index 02cb11e2a97..fb3813b7019 100644 --- a/go/vt/sqlparser/normalizer.go +++ b/go/vt/sqlparser/normalizer.go @@ -18,107 +18,277 @@ package sqlparser import ( "bytes" + "fmt" + "strconv" + "strings" "vitess.io/vitess/go/mysql/datetime" "vitess.io/vitess/go/sqltypes" + querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/sysvars" "vitess.io/vitess/go/vt/vterrors" +) - querypb "vitess.io/vitess/go/vt/proto/query" +type ( + // BindVars represents a set of reserved bind variables extracted from a SQL statement. + BindVars map[string]struct{} + // normalizer transforms SQL statements to support parameterization and streamline query planning. + // + // It serves two primary purposes: + // 1. **Parameterization:** Allows multiple invocations of the same query with different literals by converting literals + // to bind variables. This enables efficient reuse of execution plans with varying parameters. + // 2. **Simplified Planning:** Reduces the complexity for the query planner by standardizing SQL patterns. For example, + // it ensures that table columns are consistently placed on the left side of comparison expressions. This uniformity + // minimizes the number of distinct patterns the planner must handle, enhancing planning efficiency. + normalizer struct { + bindVars map[string]*querypb.BindVariable + reserved *ReservedVars + vals map[Literal]string + err error + inDerived int + inSelect int + + bindVarNeeds *BindVarNeeds + shouldRewriteDatabaseFunc bool + hasStarInSelect bool + + keyspace string + selectLimit int + setVarComment string + fkChecksState *bool + sysVars map[string]string + views VSchemaViews + + onLeave map[*AliasedExpr]func(*AliasedExpr) + parameterize bool + } + // RewriteASTResult holds the result of rewriting the AST, including bind variable needs. + RewriteASTResult struct { + *BindVarNeeds + AST Statement // The rewritten AST + } + // VSchemaViews provides access to view definitions within the VSchema. + VSchemaViews interface { + FindView(name TableName) TableStatement + } ) -// BindVars is a set of reserved bind variables from a SQL statement -type BindVars map[string]struct{} +const ( + // SQLSelectLimitUnset indicates that sql_select_limit is not set. + SQLSelectLimitUnset = -1 + // LastInsertIDName is the bind variable name for LAST_INSERT_ID(). + LastInsertIDName = "__lastInsertId" + // DBVarName is the bind variable name for DATABASE(). + DBVarName = "__vtdbname" + // FoundRowsName is the bind variable name for FOUND_ROWS(). + FoundRowsName = "__vtfrows" + // RowCountName is the bind variable name for ROW_COUNT(). + RowCountName = "__vtrcount" + // UserDefinedVariableName is the prefix for user-defined variable bind names. + UserDefinedVariableName = "__vtudv" +) -// Normalize changes the statement to use bind values, and -// updates the bind vars to those values. The supplied prefix -// is used to generate the bind var names. The function ensures -// that there are no collisions with existing bind vars. -// Within Select constructs, bind vars are deduped. This allows -// us to identify vindex equality. Otherwise, every value is -// treated as distinct. -func Normalize(stmt Statement, reserved *ReservedVars, bindVars map[string]*querypb.BindVariable) error { - nz := newNormalizer(reserved, bindVars) - _ = SafeRewrite(stmt, nz.walkStatementDown, nz.walkStatementUp) - return nz.err +// funcRewrites lists all functions that must be rewritten. we don't want these to make it down to mysql, +// we need to handle these in the vtgate +var funcRewrites = map[string]string{ + "last_insert_id": LastInsertIDName, + "database": DBVarName, + "schema": DBVarName, + "found_rows": FoundRowsName, + "row_count": RowCountName, } -type normalizer struct { - bindVars map[string]*querypb.BindVariable - reserved *ReservedVars - vals map[Literal]string - err error - inDerived int - inSelect int -} +// PrepareAST normalizes the input SQL statement and returns the rewritten AST along with bind variable information. +func PrepareAST( + in Statement, + reservedVars *ReservedVars, + bindVars map[string]*querypb.BindVariable, + parameterize bool, + keyspace string, + selectLimit int, + setVarComment string, + sysVars map[string]string, + fkChecksState *bool, + views VSchemaViews, +) (*RewriteASTResult, error) { + nz := newNormalizer(reservedVars, bindVars, keyspace, selectLimit, setVarComment, sysVars, fkChecksState, views, parameterize) + nz.shouldRewriteDatabaseFunc = shouldRewriteDatabaseFunc(in) + out := SafeRewrite(in, nz.walkDown, nz.walkUp) + if nz.err != nil { + return nil, nz.err + } -func newNormalizer(reserved *ReservedVars, bindVars map[string]*querypb.BindVariable) *normalizer { - return &normalizer{ - bindVars: bindVars, - reserved: reserved, - vals: make(map[Literal]string), + r := &RewriteASTResult{ + AST: out.(Statement), + BindVarNeeds: nz.bindVarNeeds, } + return r, nil } -// walkStatementUp is one half of the top level walk function. -func (nz *normalizer) walkStatementUp(cursor *Cursor) bool { - if nz.err != nil { - return false - } - switch node := cursor.node.(type) { - case *DerivedTable: - nz.inDerived-- - case *Select: - nz.inSelect-- - case *Literal: - if nz.inSelect == 0 { - nz.convertLiteral(node, cursor) - return nz.err == nil - } - parent := cursor.Parent() - switch parent.(type) { - case *Order, *GroupBy: - return true - case *Limit: - nz.convertLiteral(node, cursor) - default: - nz.convertLiteralDedup(node, cursor) - } +func newNormalizer( + reserved *ReservedVars, + bindVars map[string]*querypb.BindVariable, + keyspace string, + selectLimit int, + setVarComment string, + sysVars map[string]string, + fkChecksState *bool, + views VSchemaViews, + parameterize bool, +) *normalizer { + return &normalizer{ + bindVars: bindVars, + reserved: reserved, + vals: make(map[Literal]string), + bindVarNeeds: &BindVarNeeds{}, + keyspace: keyspace, + selectLimit: selectLimit, + setVarComment: setVarComment, + fkChecksState: fkChecksState, + sysVars: sysVars, + views: views, + onLeave: make(map[*AliasedExpr]func(*AliasedExpr)), + parameterize: parameterize, } - return nz.err == nil // only continue if we haven't found any errors } -// walkStatementDown is the top level walk function. -// If it encounters a Select, it switches to a mode -// where variables are deduped. -func (nz *normalizer) walkStatementDown(node, _ SQLNode) bool { +// walkDown processes nodes when traversing down the AST. +// It handles normalization logic based on node types. +func (nz *normalizer) walkDown(node, _ SQLNode) bool { switch node := node.(type) { - // no need to normalize the statement types - case *Set, *Show, *Begin, *Commit, *Rollback, *Savepoint, DDLStatement, *SRollback, *Release, *OtherAdmin, *Analyze: + case *Begin, *Commit, *Rollback, *Savepoint, *SRollback, *Release, *OtherAdmin, *Analyze, *AssignmentExpr, + *PrepareStmt, *ExecuteStmt, *FramePoint, *ColName, TableName, *ConvertType: + // These statement don't need normalizing return false + case *Set: + // Disable parameterization within SET statements. + nz.parameterize = false case *DerivedTable: nz.inDerived++ case *Select: nz.inSelect++ - case SelectExprs: - return nz.inDerived == 0 + if nz.selectLimit > 0 && node.Limit == nil { + node.Limit = &Limit{Rowcount: NewIntLiteral(strconv.Itoa(nz.selectLimit))} + } + case *AliasedExpr: + nz.noteAliasedExprName(node) case *ComparisonExpr: nz.convertComparison(node) case *UpdateExpr: nz.convertUpdateExpr(node) - case *ColName, TableName: - // Common node types that never contain Literal or ListArgs but create a lot of object - // allocations. - return false - case *ConvertType: // we should not rewrite the type description + case *StarExpr: + nz.hasStarInSelect = true + // No rewriting needed for prepare or execute statements. return false - case *FramePoint: - // do not make a bind var for rows and range + case *ShowBasic: + if node.Command != VariableGlobal && node.Command != VariableSession { + break + } + varsToAdd := sysvars.GetInterestingVariables() + for _, sysVar := range varsToAdd { + nz.bindVarNeeds.AddSysVar(sysVar) + } + } + b := nz.err == nil + if !b { + fmt.Println(1) + } + return b +} + +// noteAliasedExprName tracks expressions without aliases to add alias if expression is rewritten +func (nz *normalizer) noteAliasedExprName(node *AliasedExpr) { + if node.As.NotEmpty() { + return + } + buf := NewTrackedBuffer(nil) + node.Expr.Format(buf) + rewrites := nz.bindVarNeeds.NumberOfRewrites() + nz.onLeave[node] = func(newAliasedExpr *AliasedExpr) { + if nz.bindVarNeeds.NumberOfRewrites() > rewrites { + newAliasedExpr.As = NewIdentifierCI(buf.String()) + } + } +} + +// walkUp processes nodes when traversing up the AST. +// It finalizes normalization logic based on node types. +func (nz *normalizer) walkUp(cursor *Cursor) bool { + // Add SET_VAR comments if applicable. + if supportOptimizerHint, supports := cursor.Node().(SupportOptimizerHint); supports { + if nz.setVarComment != "" { + newComments, err := supportOptimizerHint.GetParsedComments().AddQueryHint(nz.setVarComment) + if err != nil { + nz.err = err + return false + } + supportOptimizerHint.SetComments(newComments) + } + if nz.fkChecksState != nil { + newComments := supportOptimizerHint.GetParsedComments().SetMySQLSetVarValue(sysvars.ForeignKeyChecks, FkChecksStateString(nz.fkChecksState)) + supportOptimizerHint.SetComments(newComments) + } + } + + if nz.err != nil { return false } - return nz.err == nil // only continue if we haven't found any errors + + switch node := cursor.node.(type) { + case *DerivedTable: + nz.inDerived-- + case *Select: + nz.inSelect-- + case *AliasedExpr: + // if we are tracking this node for changes, this is the time to add the alias if needed + if onLeave, ok := nz.onLeave[node]; ok { + onLeave(node) + delete(nz.onLeave, node) + } + case *Union: + nz.rewriteUnion(node) + case *FuncExpr: + nz.funcRewrite(cursor, node) + case *Variable: + nz.rewriteVariable(cursor, node) + case *Subquery: + nz.unnestSubQueries(cursor, node) + case *NotExpr: + nz.rewriteNotExpr(cursor, node) + case *AliasedTableExpr: + nz.rewriteAliasedTable(cursor, node) + case *ShowBasic: + nz.rewriteShowBasic(node) + case *ExistsExpr: + nz.existsRewrite(cursor, node) + case DistinctableAggr: + nz.rewriteDistinctableAggr(node) + case *Literal: + nz.visitLiteral(cursor, node) + } + return nz.err == nil } +func (nz *normalizer) visitLiteral(cursor *Cursor, node *Literal) { + if !nz.shouldParameterize() { + return + } + if nz.inSelect == 0 { + nz.convertLiteral(node, cursor) + return + } + switch cursor.Parent().(type) { + case *Order, *GroupBy: + return + case *Limit: + nz.convertLiteral(node, cursor) + default: + nz.convertLiteralDedup(node, cursor) + } +} + +// validateLiteral ensures that a Literal node has a valid value based on its type. func validateLiteral(node *Literal) error { switch node.Type { case DateVal: @@ -137,37 +307,31 @@ func validateLiteral(node *Literal) error { return nil } +// convertLiteralDedup converts a Literal node to a bind variable with deduplication. func (nz *normalizer) convertLiteralDedup(node *Literal, cursor *Cursor) { - err := validateLiteral(node) - if err != nil { + if err := validateLiteral(node); err != nil { nz.err = err + return } - // If value is too long, don't dedup. - // Such values are most likely not for vindexes. - // We save a lot of CPU because we avoid building - // the key for them. + // Skip deduplication for long values. if len(node.Val) > 256 { nz.convertLiteral(node, cursor) return } - // Make the bindvar - bval := SQLToBindvar(node) + bval := literalToBindvar(node) if bval == nil { return } - // Check if there's a bindvar for that value already. - bvname, ok := nz.vals[*node] - if !ok { - // If there's no such bindvar, make a new one. + bvname, exists := nz.vals[*node] + if !exists { bvname = nz.reserved.nextUnusedVar() nz.vals[*node] = bvname nz.bindVars[bvname] = bval } - // Modify the AST node to a bindvar. arg, err := NewTypedArgumentFromLiteral(bvname, node) if err != nil { nz.err = err @@ -176,14 +340,14 @@ func (nz *normalizer) convertLiteralDedup(node *Literal, cursor *Cursor) { cursor.Replace(arg) } -// convertLiteral converts an Literal without the dedup. +// convertLiteral converts a Literal node to a bind variable without deduplication. func (nz *normalizer) convertLiteral(node *Literal, cursor *Cursor) { - err := validateLiteral(node) - if err != nil { + if err := validateLiteral(node); err != nil { nz.err = err + return } - bval := SQLToBindvar(node) + bval := literalToBindvar(node) if bval == nil { return } @@ -198,11 +362,7 @@ func (nz *normalizer) convertLiteral(node *Literal, cursor *Cursor) { cursor.Replace(arg) } -// convertComparison attempts to convert IN clauses to -// use the list bind var construct. If it fails, it returns -// with no change made. The walk function will then continue -// and iterate on converting each individual value into separate -// bind vars. +// convertComparison handles the conversion of comparison expressions to use bind variables. func (nz *normalizer) convertComparison(node *ComparisonExpr) { switch node.Operator { case InOp, NotInOp: @@ -212,14 +372,19 @@ func (nz *normalizer) convertComparison(node *ComparisonExpr) { } } +// rewriteOtherComparisons parameterizes non-IN comparison expressions. func (nz *normalizer) rewriteOtherComparisons(node *ComparisonExpr) { - newR := nz.parameterize(node.Left, node.Right) + newR := nz.normalizeComparisonWithBindVar(node.Left, node.Right) if newR != nil { node.Right = newR } } -func (nz *normalizer) parameterize(left, right Expr) Expr { +// normalizeComparisonWithBindVar attempts to replace a literal in a comparison with a bind variable. +func (nz *normalizer) normalizeComparisonWithBindVar(left, right Expr) Expr { + if !nz.shouldParameterize() { + return nil + } col, ok := left.(*ColName) if !ok { return nil @@ -228,13 +393,12 @@ func (nz *normalizer) parameterize(left, right Expr) Expr { if !ok { return nil } - err := validateLiteral(lit) - if err != nil { + if err := validateLiteral(lit); err != nil { nz.err = err return nil } - bval := SQLToBindvar(lit) + bval := literalToBindvar(lit) if bval == nil { return nil } @@ -247,18 +411,14 @@ func (nz *normalizer) parameterize(left, right Expr) Expr { return arg } +// decideBindVarName determines the appropriate bind variable name for a given literal and column. func (nz *normalizer) decideBindVarName(lit *Literal, col *ColName, bval *querypb.BindVariable) string { if len(lit.Val) <= 256 { - // first we check if we already have a bindvar for this value. if we do, we re-use that bindvar name - bvname, ok := nz.vals[*lit] - if ok { + if bvname, ok := nz.vals[*lit]; ok { return bvname } } - // If there's no such bindvar, or we have a big value, make a new one. - // Big values are most likely not for vindexes. - // We save a lot of CPU because we avoid building bvname := nz.reserved.ReserveColName(col) nz.vals[*lit] = bvname nz.bindVars[bvname] = bval @@ -266,19 +426,22 @@ func (nz *normalizer) decideBindVarName(lit *Literal, col *ColName, bval *queryp return bvname } +// rewriteInComparisons converts IN and NOT IN expressions to use list bind variables. func (nz *normalizer) rewriteInComparisons(node *ComparisonExpr) { + if !nz.shouldParameterize() { + return + } tupleVals, ok := node.Right.(ValTuple) if !ok { return } - // The RHS is a tuple of values. - // Make a list bindvar. + // Create a list bind variable for the tuple. bvals := &querypb.BindVariable{ Type: querypb.Type_TUPLE, } for _, val := range tupleVals { - bval := SQLToBindvar(val) + bval := literalToBindvar(val) if bval == nil { return } @@ -289,76 +452,74 @@ func (nz *normalizer) rewriteInComparisons(node *ComparisonExpr) { } bvname := nz.reserved.nextUnusedVar() nz.bindVars[bvname] = bvals - // Modify RHS to be a list bindvar. node.Right = ListArg(bvname) } +// convertUpdateExpr parameterizes expressions in UPDATE statements. func (nz *normalizer) convertUpdateExpr(node *UpdateExpr) { - newR := nz.parameterize(node.Name, node.Expr) + newR := nz.normalizeComparisonWithBindVar(node.Name, node.Expr) if newR != nil { node.Expr = newR } } -func SQLToBindvar(node SQLNode) *querypb.BindVariable { - if node, ok := node.(*Literal); ok { - var v sqltypes.Value - var err error - switch node.Type { - case StrVal: - v, err = sqltypes.NewValue(sqltypes.VarChar, node.Bytes()) - case IntVal: - v, err = sqltypes.NewValue(sqltypes.Int64, node.Bytes()) - case FloatVal: - v, err = sqltypes.NewValue(sqltypes.Float64, node.Bytes()) - case DecimalVal: - v, err = sqltypes.NewValue(sqltypes.Decimal, node.Bytes()) - case HexNum: - buf := make([]byte, 0, len(node.Bytes())) - buf = append(buf, "0x"...) - buf = append(buf, bytes.ToUpper(node.Bytes()[2:])...) - v, err = sqltypes.NewValue(sqltypes.HexNum, buf) - case HexVal: - // We parse the `x'7b7d'` string literal into a hex encoded string of `7b7d` in the parser - // We need to re-encode it back to the original MySQL query format before passing it on as a bindvar value to MySQL - buf := make([]byte, 0, len(node.Bytes())+3) - buf = append(buf, 'x', '\'') - buf = append(buf, bytes.ToUpper(node.Bytes())...) - buf = append(buf, '\'') - v, err = sqltypes.NewValue(sqltypes.HexVal, buf) - case BitNum: - out := make([]byte, 0, len(node.Bytes())+2) - out = append(out, '0', 'b') - out = append(out, node.Bytes()[2:]...) - v, err = sqltypes.NewValue(sqltypes.BitNum, out) - case DateVal: - v, err = sqltypes.NewValue(sqltypes.Date, node.Bytes()) - case TimeVal: - v, err = sqltypes.NewValue(sqltypes.Time, node.Bytes()) - case TimestampVal: - // This is actually a DATETIME MySQL type. The timestamp literal - // syntax is part of the SQL standard and MySQL DATETIME matches - // the type best. - v, err = sqltypes.NewValue(sqltypes.Datetime, node.Bytes()) - default: - return nil - } - if err != nil { - return nil - } - return sqltypes.ValueBindVariable(v) +// literalToBindvar converts a SQLNode to a BindVariable if possible. +func literalToBindvar(node SQLNode) *querypb.BindVariable { + lit, ok := node.(*Literal) + if !ok { + return nil } - return nil + var v sqltypes.Value + var err error + switch lit.Type { + case StrVal: + v, err = sqltypes.NewValue(sqltypes.VarChar, lit.Bytes()) + case IntVal: + v, err = sqltypes.NewValue(sqltypes.Int64, lit.Bytes()) + case FloatVal: + v, err = sqltypes.NewValue(sqltypes.Float64, lit.Bytes()) + case DecimalVal: + v, err = sqltypes.NewValue(sqltypes.Decimal, lit.Bytes()) + case HexNum: + buf := make([]byte, 0, len(lit.Bytes())) + buf = append(buf, "0x"...) + buf = append(buf, bytes.ToUpper(lit.Bytes()[2:])...) + v, err = sqltypes.NewValue(sqltypes.HexNum, buf) + case HexVal: + // Re-encode hex string literals to original MySQL format. + buf := make([]byte, 0, len(lit.Bytes())+3) + buf = append(buf, 'x', '\'') + buf = append(buf, bytes.ToUpper(lit.Bytes())...) + buf = append(buf, '\'') + v, err = sqltypes.NewValue(sqltypes.HexVal, buf) + case BitNum: + out := make([]byte, 0, len(lit.Bytes())+2) + out = append(out, '0', 'b') + out = append(out, lit.Bytes()[2:]...) + v, err = sqltypes.NewValue(sqltypes.BitNum, out) + case DateVal: + v, err = sqltypes.NewValue(sqltypes.Date, lit.Bytes()) + case TimeVal: + v, err = sqltypes.NewValue(sqltypes.Time, lit.Bytes()) + case TimestampVal: + // Use DATETIME type for TIMESTAMP literals. + v, err = sqltypes.NewValue(sqltypes.Datetime, lit.Bytes()) + default: + return nil + } + if err != nil { + return nil + } + return sqltypes.ValueBindVariable(v) } -// GetBindvars returns a map of the bind vars referenced in the statement. -func GetBindvars(stmt Statement) map[string]struct{} { +// getBindvars extracts bind variables from a SQL statement. +func getBindvars(stmt Statement) map[string]struct{} { bindvars := make(map[string]struct{}) _ = Walk(func(node SQLNode) (kontinue bool, err error) { switch node := node.(type) { case *ColName, TableName: - // Common node types that never contain expressions but create a lot of object - // allocations. + // These node types do not contain expressions. return false, nil case *Argument: bindvars[node.Name] = struct{}{} @@ -369,3 +530,312 @@ func GetBindvars(stmt Statement) map[string]struct{} { }, stmt) return bindvars } + +var HasValueSubQueryBaseName = []byte("__sq_has_values") + +// shouldRewriteDatabaseFunc determines if the database function should be rewritten based on the statement. +func shouldRewriteDatabaseFunc(in Statement) bool { + selct, ok := in.(*Select) + if !ok { + return false + } + if len(selct.From) != 1 { + return false + } + aliasedTable, ok := selct.From[0].(*AliasedTableExpr) + if !ok { + return false + } + tableName, ok := aliasedTable.Expr.(TableName) + if !ok { + return false + } + return tableName.Name.String() == "dual" +} + +// rewriteUnion sets the SELECT limit for UNION statements if not already set. +func (nz *normalizer) rewriteUnion(node *Union) { + if nz.selectLimit > 0 && node.Limit == nil { + node.Limit = &Limit{Rowcount: NewIntLiteral(strconv.Itoa(nz.selectLimit))} + } +} + +// rewriteAliasedTable handles the rewriting of aliased tables, including view substitutions. +func (nz *normalizer) rewriteAliasedTable(cursor *Cursor, node *AliasedTableExpr) { + aliasTableName, ok := node.Expr.(TableName) + if !ok { + return + } + + // Do not add qualifiers to the dual table. + tblName := aliasTableName.Name.String() + if tblName == "dual" { + return + } + + if SystemSchema(nz.keyspace) { + if aliasTableName.Qualifier.IsEmpty() { + aliasTableName.Qualifier = NewIdentifierCS(nz.keyspace) + node.Expr = aliasTableName + cursor.Replace(node) + } + return + } + + // Replace views with their underlying definitions. + if nz.views == nil { + return + } + view := nz.views.FindView(aliasTableName) + if view == nil { + return + } + + // Substitute the view with a derived table. + node.Expr = &DerivedTable{Select: Clone(view)} + if node.As.IsEmpty() { + node.As = NewIdentifierCS(tblName) + } +} + +// rewriteShowBasic handles the rewriting of SHOW statements, particularly for system variables. +func (nz *normalizer) rewriteShowBasic(node *ShowBasic) { + if node.Command == VariableGlobal || node.Command == VariableSession { + varsToAdd := sysvars.GetInterestingVariables() + for _, sysVar := range varsToAdd { + nz.bindVarNeeds.AddSysVar(sysVar) + } + } +} + +// rewriteNotExpr simplifies NOT expressions where possible. +func (nz *normalizer) rewriteNotExpr(cursor *Cursor, node *NotExpr) { + switch inner := node.Expr.(type) { + case *ComparisonExpr: + // Invert comparison operators. + if canChange, inverse := inverseOp(inner.Operator); canChange { + inner.Operator = inverse + cursor.Replace(inner) + } + case *NotExpr: + // Simplify double negation. + cursor.Replace(inner.Expr) + case BoolVal: + // Negate boolean values. + cursor.Replace(!inner) + } +} + +// rewriteVariable handles the rewriting of variable expressions to bind variables. +func (nz *normalizer) rewriteVariable(cursor *Cursor, node *Variable) { + // Do not rewrite variables on the left side of SET assignments. + if v, isSet := cursor.Parent().(*SetExpr); isSet && v.Var == node { + return + } + switch node.Scope { + case VariableScope: + nz.udvRewrite(cursor, node) + case SessionScope, NextTxScope: + nz.sysVarRewrite(cursor, node) + } +} + +// inverseOp returns the inverse operator for a given comparison operator. +func inverseOp(i ComparisonExprOperator) (bool, ComparisonExprOperator) { + switch i { + case EqualOp: + return true, NotEqualOp + case LessThanOp: + return true, GreaterEqualOp + case GreaterThanOp: + return true, LessEqualOp + case LessEqualOp: + return true, GreaterThanOp + case GreaterEqualOp: + return true, LessThanOp + case NotEqualOp: + return true, EqualOp + case InOp: + return true, NotInOp + case NotInOp: + return true, InOp + case LikeOp: + return true, NotLikeOp + case NotLikeOp: + return true, LikeOp + case RegexpOp: + return true, NotRegexpOp + case NotRegexpOp: + return true, RegexpOp + } + return false, i +} + +// sysVarRewrite replaces system variables with corresponding bind variables. +func (nz *normalizer) sysVarRewrite(cursor *Cursor, node *Variable) { + lowered := node.Name.Lowered() + + var found bool + if nz.sysVars != nil { + _, found = nz.sysVars[lowered] + } + + switch lowered { + case sysvars.Autocommit.Name, + sysvars.Charset.Name, + sysvars.ClientFoundRows.Name, + sysvars.DDLStrategy.Name, + sysvars.MigrationContext.Name, + sysvars.Names.Name, + sysvars.TransactionMode.Name, + sysvars.ReadAfterWriteGTID.Name, + sysvars.ReadAfterWriteTimeOut.Name, + sysvars.SessionEnableSystemSettings.Name, + sysvars.SessionTrackGTIDs.Name, + sysvars.SessionUUID.Name, + sysvars.SkipQueryPlanCache.Name, + sysvars.Socket.Name, + sysvars.SQLSelectLimit.Name, + sysvars.Version.Name, + sysvars.VersionComment.Name, + sysvars.QueryTimeout.Name, + sysvars.Workload.Name: + found = true + } + + if found { + cursor.Replace(NewArgument("__vt" + lowered)) + nz.bindVarNeeds.AddSysVar(lowered) + } +} + +// udvRewrite replaces user-defined variables with corresponding bind variables. +func (nz *normalizer) udvRewrite(cursor *Cursor, node *Variable) { + udv := strings.ToLower(node.Name.CompliantName()) + cursor.Replace(NewArgument(UserDefinedVariableName + udv)) + nz.bindVarNeeds.AddUserDefVar(udv) +} + +// funcRewrite replaces certain function expressions with bind variables. +func (nz *normalizer) funcRewrite(cursor *Cursor, node *FuncExpr) { + lowered := node.Name.Lowered() + if lowered == "last_insert_id" && len(node.Exprs) > 0 { + // Do not rewrite LAST_INSERT_ID() when it has arguments. + return + } + bindVar, found := funcRewrites[lowered] + if !found || (bindVar == DBVarName && !nz.shouldRewriteDatabaseFunc) { + return + } + if len(node.Exprs) > 0 { + nz.err = vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "Argument to %s() not supported", lowered) + return + } + cursor.Replace(NewArgument(bindVar)) + nz.bindVarNeeds.AddFuncResult(bindVar) +} + +// unnestSubQueries attempts to simplify dual subqueries where possible. +// select (select database() from dual) from test +// => +// select database() from test +func (nz *normalizer) unnestSubQueries(cursor *Cursor, subquery *Subquery) { + if _, isExists := cursor.Parent().(*ExistsExpr); isExists { + return + } + sel, isSimpleSelect := subquery.Select.(*Select) + if !isSimpleSelect { + return + } + + if len(sel.SelectExprs) != 1 || + len(sel.OrderBy) != 0 || + sel.GroupBy != nil || + len(sel.From) != 1 || + sel.Where != nil || + sel.Having != nil || + sel.Limit != nil || sel.Lock != NoLock { + return + } + + aliasedTable, ok := sel.From[0].(*AliasedTableExpr) + if !ok { + return + } + table, ok := aliasedTable.Expr.(TableName) + if !ok || table.Name.String() != "dual" { + return + } + expr, ok := sel.SelectExprs[0].(*AliasedExpr) + if !ok { + return + } + _, isColName := expr.Expr.(*ColName) + if isColName { + // Skip if the subquery already returns a column name. + return + } + nz.bindVarNeeds.NoteRewrite() + rewritten := SafeRewrite(expr.Expr, nz.walkDown, nz.walkUp) + + // Handle special cases for IN clauses. + rewrittenExpr, isExpr := rewritten.(Expr) + _, isColTuple := rewritten.(ColTuple) + comparisonExpr, isCompExpr := cursor.Parent().(*ComparisonExpr) + if isCompExpr && (comparisonExpr.Operator == InOp || comparisonExpr.Operator == NotInOp) && !isColTuple && isExpr { + cursor.Replace(ValTuple{rewrittenExpr}) + return + } + + cursor.Replace(rewritten) +} + +// existsRewrite optimizes EXISTS expressions where possible. +func (nz *normalizer) existsRewrite(cursor *Cursor, node *ExistsExpr) { + sel, ok := node.Subquery.Select.(*Select) + if !ok { + return + } + + if sel.Having != nil { + // Cannot optimize if HAVING clause is present. + return + } + + if sel.GroupBy == nil && sel.SelectExprs.AllAggregation() { + // Replace EXISTS with a boolean true if guaranteed to be non-empty. + cursor.Replace(BoolVal(true)) + return + } + + // Simplify the subquery by selecting a constant. + // WHERE EXISTS(SELECT 1 FROM ...) + sel.SelectExprs = SelectExprs{ + &AliasedExpr{Expr: NewIntLiteral("1")}, + } + sel.GroupBy = nil +} + +// rewriteDistinctableAggr removes DISTINCT from certain aggregations to simplify the plan. +func (nz *normalizer) rewriteDistinctableAggr(node DistinctableAggr) { + if !node.IsDistinct() { + return + } + switch aggr := node.(type) { + case *Max, *Min: + aggr.SetDistinct(false) + nz.bindVarNeeds.NoteRewrite() + } +} + +func (nz *normalizer) shouldParameterize() bool { + return !(nz.inDerived > 0 && len(nz.onLeave) > 0) && nz.parameterize +} + +// SystemSchema checks if the given schema is a system schema. +func SystemSchema(schema string) bool { + return strings.EqualFold(schema, "information_schema") || + strings.EqualFold(schema, "performance_schema") || + strings.EqualFold(schema, "sys") || + strings.EqualFold(schema, "mysql") +} diff --git a/go/vt/sqlparser/normalizer_test.go b/go/vt/sqlparser/normalizer_test.go index 7919a321c91..7c3e660ac9d 100644 --- a/go/vt/sqlparser/normalizer_test.go +++ b/go/vt/sqlparser/normalizer_test.go @@ -25,8 +25,9 @@ import ( "strings" "testing" - "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/vt/sysvars" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "vitess.io/vitess/go/sqltypes" @@ -448,10 +449,11 @@ func TestNormalize(t *testing.T) { t.Run(tc.in, func(t *testing.T) { stmt, err := parser.Parse(tc.in) require.NoError(t, err) - known := GetBindvars(stmt) + known := getBindvars(stmt) bv := make(map[string]*querypb.BindVariable) - require.NoError(t, Normalize(stmt, NewReservedVars(prefix, known), bv)) - assert.Equal(t, tc.outstmt, String(stmt)) + out, err := PrepareAST(stmt, NewReservedVars(prefix, known), bv, true, "ks", 0, "", map[string]string{}, nil, nil) + require.NoError(t, err) + assert.Equal(t, tc.outstmt, String(out.AST)) assert.Equal(t, tc.outbv, bv) }) } @@ -476,9 +478,10 @@ func TestNormalizeInvalidDates(t *testing.T) { t.Run(tc.in, func(t *testing.T) { stmt, err := parser.Parse(tc.in) require.NoError(t, err) - known := GetBindvars(stmt) + known := getBindvars(stmt) bv := make(map[string]*querypb.BindVariable) - require.EqualError(t, Normalize(stmt, NewReservedVars("bv", known), bv), tc.err.Error()) + _, err = PrepareAST(stmt, NewReservedVars("bv", known), bv, true, "ks", 0, "", map[string]string{}, nil, nil) + require.EqualError(t, err, tc.err.Error()) }) } } @@ -498,9 +501,10 @@ func TestNormalizeValidSQL(t *testing.T) { } bv := make(map[string]*querypb.BindVariable) known := make(BindVars) - err = Normalize(tree, NewReservedVars("vtg", known), bv) + + out, err := PrepareAST(tree, NewReservedVars("vtg", known), bv, true, "ks", 0, "", map[string]string{}, nil, nil) require.NoError(t, err) - normalizerOutput := String(tree) + normalizerOutput := String(out.AST) if normalizerOutput == "otheradmin" || normalizerOutput == "otherread" { return } @@ -529,9 +533,9 @@ func TestNormalizeOneCasae(t *testing.T) { } bv := make(map[string]*querypb.BindVariable) known := make(BindVars) - err = Normalize(tree, NewReservedVars("vtg", known), bv) + out, err := PrepareAST(tree, NewReservedVars("vtg", known), bv, true, "ks", 0, "", map[string]string{}, nil, nil) require.NoError(t, err) - normalizerOutput := String(tree) + normalizerOutput := String(out.AST) require.EqualValues(t, testOne.output, normalizerOutput) if normalizerOutput == "otheradmin" || normalizerOutput == "otherread" { return @@ -546,7 +550,7 @@ func TestGetBindVars(t *testing.T) { if err != nil { t.Fatal(err) } - got := GetBindvars(stmt) + got := getBindvars(stmt) want := map[string]struct{}{ "v1": {}, "v2": {}, @@ -559,6 +563,586 @@ func TestGetBindVars(t *testing.T) { } } +type testCaseSetVar struct { + in, expected, setVarComment string +} + +type testCaseSysVar struct { + in, expected string + sysVar map[string]string +} + +type myTestCase struct { + in, expected string + liid, db, foundRows, rowCount, rawGTID, rawTimeout, sessTrackGTID bool + ddlStrategy, migrationContext, sessionUUID, sessionEnableSystemSettings bool + udv int + autocommit, foreignKeyChecks, clientFoundRows, skipQueryPlanCache, socket, queryTimeout bool + sqlSelectLimit, transactionMode, workload, version, versionComment bool +} + +func TestRewrites(in *testing.T) { + tests := []myTestCase{{ + in: "SELECT 42", + expected: "SELECT 42", + // no bindvar needs + }, { + in: "SELECT @@version", + expected: "SELECT :__vtversion as `@@version`", + version: true, + }, { + in: "SELECT @@query_timeout", + expected: "SELECT :__vtquery_timeout as `@@query_timeout`", + queryTimeout: true, + }, { + in: "SELECT @@version_comment", + expected: "SELECT :__vtversion_comment as `@@version_comment`", + versionComment: true, + }, { + in: "SELECT @@enable_system_settings", + expected: "SELECT :__vtenable_system_settings as `@@enable_system_settings`", + sessionEnableSystemSettings: true, + }, { + in: "SELECT last_insert_id()", + expected: "SELECT :__lastInsertId as `last_insert_id()`", + liid: true, + }, { + in: "SELECT database()", + expected: "SELECT :__vtdbname as `database()`", + db: true, + }, { + in: "SELECT database() from test", + expected: "SELECT database() from test", + // no bindvar needs + }, { + in: "SELECT last_insert_id() as test", + expected: "SELECT :__lastInsertId as test", + liid: true, + }, { + in: "SELECT last_insert_id() + database()", + expected: "SELECT :__lastInsertId + :__vtdbname as `last_insert_id() + database()`", + db: true, liid: true, + }, { + // unnest database() call + in: "select (select database()) from test", + expected: "select database() as `(select database() from dual)` from test", + // no bindvar needs + }, { + // unnest database() call + in: "select (select database() from dual) from test", + expected: "select database() as `(select database() from dual)` from test", + // no bindvar needs + }, { + in: "select (select database() from dual) from dual", + expected: "select :__vtdbname as `(select database() from dual)` from dual", + db: true, + }, { + // don't unnest solo columns + in: "select 1 as foobar, (select foobar)", + expected: "select 1 as foobar, (select foobar from dual) from dual", + }, { + in: "select id from user where database()", + expected: "select id from user where database()", + // no bindvar needs + }, { + in: "select table_name from information_schema.tables where table_schema = database()", + expected: "select table_name from information_schema.tables where table_schema = database()", + // no bindvar needs + }, { + in: "select schema()", + expected: "select :__vtdbname as `schema()`", + db: true, + }, { + in: "select found_rows()", + expected: "select :__vtfrows as `found_rows()`", + foundRows: true, + }, { + in: "select @`x y`", + expected: "select :__vtudvx_y as `@``x y``` from dual", + udv: 1, + }, { + in: "select id from t where id = @x and val = @y", + expected: "select id from t where id = :__vtudvx and val = :__vtudvy", + db: false, udv: 2, + }, { + in: "insert into t(id) values(@xyx)", + expected: "insert into t(id) values(:__vtudvxyx)", + db: false, udv: 1, + }, { + in: "select row_count()", + expected: "select :__vtrcount as `row_count()`", + rowCount: true, + }, { + in: "SELECT lower(database())", + expected: "SELECT lower(:__vtdbname) as `lower(database())`", + db: true, + }, { + in: "SELECT @@autocommit", + expected: "SELECT :__vtautocommit as `@@autocommit`", + autocommit: true, + }, { + in: "SELECT @@client_found_rows", + expected: "SELECT :__vtclient_found_rows as `@@client_found_rows`", + clientFoundRows: true, + }, { + in: "SELECT @@skip_query_plan_cache", + expected: "SELECT :__vtskip_query_plan_cache as `@@skip_query_plan_cache`", + skipQueryPlanCache: true, + }, { + in: "SELECT @@sql_select_limit", + expected: "SELECT :__vtsql_select_limit as `@@sql_select_limit`", + sqlSelectLimit: true, + }, { + in: "SELECT @@transaction_mode", + expected: "SELECT :__vttransaction_mode as `@@transaction_mode`", + transactionMode: true, + }, { + in: "SELECT @@workload", + expected: "SELECT :__vtworkload as `@@workload`", + workload: true, + }, { + in: "SELECT @@socket", + expected: "SELECT :__vtsocket as `@@socket`", + socket: true, + }, { + in: "select (select 42) from dual", + expected: "select 42 as `(select 42 from dual)` from dual", + }, { + in: "select * from user where col = (select 42)", + expected: "select * from user where col = 42", + }, { + in: "select * from (select 42) as t", // this is not an expression, and should not be rewritten + expected: "select * from (select 42) as t", + }, { + in: `select (select (select (select (select (select last_insert_id()))))) as x`, + expected: "select :__lastInsertId as x from dual", + liid: true, + }, { + in: `select * from (select last_insert_id()) as t`, + expected: "select * from (select :__lastInsertId as `last_insert_id()` from dual) as t", + liid: true, + }, { + in: `select * from user where col = @@ddl_strategy`, + expected: "select * from user where col = :__vtddl_strategy", + ddlStrategy: true, + }, { + in: `select * from user where col = @@migration_context`, + expected: "select * from user where col = :__vtmigration_context", + migrationContext: true, + }, { + in: `select * from user where col = @@read_after_write_gtid OR col = @@read_after_write_timeout OR col = @@session_track_gtids`, + expected: "select * from user where col = :__vtread_after_write_gtid or col = :__vtread_after_write_timeout or col = :__vtsession_track_gtids", + rawGTID: true, rawTimeout: true, sessTrackGTID: true, + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual)", + expected: "SELECT * FROM tbl WHERE id IN (1)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT last_insert_id() FROM dual)", + expected: "SELECT * FROM tbl WHERE id IN (:__lastInsertId)", + liid: true, + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT (SELECT 1 FROM dual WHERE 1 = 0) FROM dual)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1,2 FROM dual)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1,2 FROM dual)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual ORDER BY 1)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual ORDER BY 1)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT id FROM user GROUP BY id)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT id FROM user GROUP BY id)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual, user)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual, user)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual limit 1)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual limit 1)", + }, { + // SELECT * behaves different depending the join type used, so if that has been used, we won't rewrite + in: "SELECT * FROM A JOIN B USING (id1,id2,id3)", + expected: "SELECT * FROM A JOIN B USING (id1,id2,id3)", + }, { + in: "CALL proc(@foo)", + expected: "CALL proc(:__vtudvfoo)", + udv: 1, + }, { + in: "SELECT * FROM tbl WHERE NOT id = 42", + expected: "SELECT * FROM tbl WHERE id != 42", + }, { + in: "SELECT * FROM tbl WHERE not id < 12", + expected: "SELECT * FROM tbl WHERE id >= 12", + }, { + in: "SELECT * FROM tbl WHERE not id > 12", + expected: "SELECT * FROM tbl WHERE id <= 12", + }, { + in: "SELECT * FROM tbl WHERE not id <= 33", + expected: "SELECT * FROM tbl WHERE id > 33", + }, { + in: "SELECT * FROM tbl WHERE not id >= 33", + expected: "SELECT * FROM tbl WHERE id < 33", + }, { + in: "SELECT * FROM tbl WHERE not id != 33", + expected: "SELECT * FROM tbl WHERE id = 33", + }, { + in: "SELECT * FROM tbl WHERE not id in (1,2,3)", + expected: "SELECT * FROM tbl WHERE id not in (1,2,3)", + }, { + in: "SELECT * FROM tbl WHERE not id not in (1,2,3)", + expected: "SELECT * FROM tbl WHERE id in (1,2,3)", + }, { + in: "SELECT * FROM tbl WHERE not id not in (1,2,3)", + expected: "SELECT * FROM tbl WHERE id in (1,2,3)", + }, { + in: "SELECT * FROM tbl WHERE not id like '%foobar'", + expected: "SELECT * FROM tbl WHERE id not like '%foobar'", + }, { + in: "SELECT * FROM tbl WHERE not id not like '%foobar'", + expected: "SELECT * FROM tbl WHERE id like '%foobar'", + }, { + in: "SELECT * FROM tbl WHERE not id regexp '%foobar'", + expected: "SELECT * FROM tbl WHERE id not regexp '%foobar'", + }, { + in: "SELECT * FROM tbl WHERE not id not regexp '%foobar'", + expected: "select * from tbl where id regexp '%foobar'", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar)", + expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar limit 100 offset 34)", + expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar limit 100 offset 34)", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2)", + expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar group by col1, col2)", + expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", + }, { + in: "SELECT * FROM tbl WHERE exists(select count(*) from other_table where foo > bar)", + expected: "SELECT * FROM tbl WHERE true", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2 having count(*) > 3)", + expected: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2 having count(*) > 3)", + }, { + in: "SELECT id, name, salary FROM user_details", + expected: "SELECT id, name, salary FROM (select user.id, user.name, user_extra.salary from user join user_extra where user.id = user_extra.user_id) as user_details", + }, { + in: "select max(distinct c1), min(distinct c2), avg(distinct c3), sum(distinct c4), count(distinct c5), group_concat(distinct c6) from tbl", + expected: "select max(c1) as `max(distinct c1)`, min(c2) as `min(distinct c2)`, avg(distinct c3), sum(distinct c4), count(distinct c5), group_concat(distinct c6) from tbl", + }, { + in: "SHOW VARIABLES", + expected: "SHOW VARIABLES", + autocommit: true, + foreignKeyChecks: true, + clientFoundRows: true, + skipQueryPlanCache: true, + sqlSelectLimit: true, + transactionMode: true, + workload: true, + version: true, + versionComment: true, + ddlStrategy: true, + migrationContext: true, + sessionUUID: true, + sessionEnableSystemSettings: true, + rawGTID: true, + rawTimeout: true, + sessTrackGTID: true, + socket: true, + queryTimeout: true, + }, { + in: "SHOW GLOBAL VARIABLES", + expected: "SHOW GLOBAL VARIABLES", + autocommit: true, + foreignKeyChecks: true, + clientFoundRows: true, + skipQueryPlanCache: true, + sqlSelectLimit: true, + transactionMode: true, + workload: true, + version: true, + versionComment: true, + ddlStrategy: true, + migrationContext: true, + sessionUUID: true, + sessionEnableSystemSettings: true, + rawGTID: true, + rawTimeout: true, + sessTrackGTID: true, + socket: true, + queryTimeout: true, + }} + parser := NewTestParser() + for _, tc := range tests { + in.Run(tc.in, func(t *testing.T) { + require := require.New(t) + stmt, known, err := parser.Parse2(tc.in) + require.NoError(err) + vars := NewReservedVars("v", known) + result, err := PrepareAST( + stmt, + vars, + map[string]*querypb.BindVariable{}, + false, + "ks", + 0, + "", + map[string]string{}, + nil, + &fakeViews{}, + ) + require.NoError(err) + + expected, err := parser.Parse(tc.expected) + require.NoError(err, "test expectation does not parse [%s]", tc.expected) + + s := String(expected) + assert := assert.New(t) + assert.Equal(s, String(result.AST)) + assert.Equal(tc.liid, result.NeedsFuncResult(LastInsertIDName), "should need last insert id") + assert.Equal(tc.db, result.NeedsFuncResult(DBVarName), "should need database name") + assert.Equal(tc.foundRows, result.NeedsFuncResult(FoundRowsName), "should need found rows") + assert.Equal(tc.rowCount, result.NeedsFuncResult(RowCountName), "should need row count") + assert.Equal(tc.udv, len(result.NeedUserDefinedVariables), "count of user defined variables") + assert.Equal(tc.autocommit, result.NeedsSysVar(sysvars.Autocommit.Name), "should need :__vtautocommit") + assert.Equal(tc.foreignKeyChecks, result.NeedsSysVar(sysvars.ForeignKeyChecks), "should need :__vtforeignKeyChecks") + assert.Equal(tc.clientFoundRows, result.NeedsSysVar(sysvars.ClientFoundRows.Name), "should need :__vtclientFoundRows") + assert.Equal(tc.skipQueryPlanCache, result.NeedsSysVar(sysvars.SkipQueryPlanCache.Name), "should need :__vtskipQueryPlanCache") + assert.Equal(tc.sqlSelectLimit, result.NeedsSysVar(sysvars.SQLSelectLimit.Name), "should need :__vtsqlSelectLimit") + assert.Equal(tc.transactionMode, result.NeedsSysVar(sysvars.TransactionMode.Name), "should need :__vttransactionMode") + assert.Equal(tc.workload, result.NeedsSysVar(sysvars.Workload.Name), "should need :__vtworkload") + assert.Equal(tc.queryTimeout, result.NeedsSysVar(sysvars.QueryTimeout.Name), "should need :__vtquery_timeout") + assert.Equal(tc.ddlStrategy, result.NeedsSysVar(sysvars.DDLStrategy.Name), "should need ddlStrategy") + assert.Equal(tc.migrationContext, result.NeedsSysVar(sysvars.MigrationContext.Name), "should need migrationContext") + assert.Equal(tc.sessionUUID, result.NeedsSysVar(sysvars.SessionUUID.Name), "should need sessionUUID") + assert.Equal(tc.sessionEnableSystemSettings, result.NeedsSysVar(sysvars.SessionEnableSystemSettings.Name), "should need sessionEnableSystemSettings") + assert.Equal(tc.rawGTID, result.NeedsSysVar(sysvars.ReadAfterWriteGTID.Name), "should need rawGTID") + assert.Equal(tc.rawTimeout, result.NeedsSysVar(sysvars.ReadAfterWriteTimeOut.Name), "should need rawTimeout") + assert.Equal(tc.sessTrackGTID, result.NeedsSysVar(sysvars.SessionTrackGTIDs.Name), "should need sessTrackGTID") + assert.Equal(tc.version, result.NeedsSysVar(sysvars.Version.Name), "should need Vitess version") + assert.Equal(tc.versionComment, result.NeedsSysVar(sysvars.VersionComment.Name), "should need Vitess version") + assert.Equal(tc.socket, result.NeedsSysVar(sysvars.Socket.Name), "should need :__vtsocket") + }) + } +} + +type fakeViews struct{} + +func (*fakeViews) FindView(name TableName) TableStatement { + if name.Name.String() != "user_details" { + return nil + } + parser := NewTestParser() + statement, err := parser.Parse("select user.id, user.name, user_extra.salary from user join user_extra where user.id = user_extra.user_id") + if err != nil { + return nil + } + return statement.(TableStatement) +} + +func TestRewritesWithSetVarComment(in *testing.T) { + tests := []testCaseSetVar{{ + in: "select 1", + expected: "select 1", + setVarComment: "", + }, { + in: "select 1", + expected: "select /*+ AA(a) */ 1", + setVarComment: "AA(a)", + }, { + in: "insert /* toto */ into t(id) values(1)", + expected: "insert /*+ AA(a) */ /* toto */ into t(id) values(1)", + setVarComment: "AA(a)", + }, { + in: "select /* toto */ * from t union select * from s", + expected: "select /*+ AA(a) */ /* toto */ * from t union select /*+ AA(a) */ * from s", + setVarComment: "AA(a)", + }, { + in: "vstream /* toto */ * from t1", + expected: "vstream /*+ AA(a) */ /* toto */ * from t1", + setVarComment: "AA(a)", + }, { + in: "stream /* toto */ t from t1", + expected: "stream /*+ AA(a) */ /* toto */ t from t1", + setVarComment: "AA(a)", + }, { + in: "update /* toto */ t set id = 1", + expected: "update /*+ AA(a) */ /* toto */ t set id = 1", + setVarComment: "AA(a)", + }, { + in: "delete /* toto */ from t", + expected: "delete /*+ AA(a) */ /* toto */ from t", + setVarComment: "AA(a)", + }} + + parser := NewTestParser() + for _, tc := range tests { + in.Run(tc.in, func(t *testing.T) { + require := require.New(t) + stmt, err := parser.Parse(tc.in) + require.NoError(err) + vars := NewReservedVars("v", nil) + result, err := PrepareAST( + stmt, + vars, + map[string]*querypb.BindVariable{}, + false, + "ks", + 0, + tc.setVarComment, + map[string]string{}, + nil, + &fakeViews{}, + ) + + require.NoError(err) + + expected, err := parser.Parse(tc.expected) + require.NoError(err, "test expectation does not parse [%s]", tc.expected) + + assert.Equal(t, String(expected), String(result.AST)) + }) + } +} + +func TestRewritesSysVar(in *testing.T) { + tests := []testCaseSysVar{{ + in: "select @x = @@sql_mode", + expected: "select :__vtudvx = @@sql_mode as `@x = @@sql_mode` from dual", + }, { + in: "select @x = @@sql_mode", + expected: "select :__vtudvx = :__vtsql_mode as `@x = @@sql_mode` from dual", + sysVar: map[string]string{"sql_mode": "' '"}, + }, { + in: "SELECT @@tx_isolation", + expected: "select @@tx_isolation from dual", + }, { + in: "SELECT @@transaction_isolation", + expected: "select @@transaction_isolation from dual", + }, { + in: "SELECT @@session.transaction_isolation", + expected: "select @@session.transaction_isolation from dual", + }, { + in: "SELECT @@tx_isolation", + sysVar: map[string]string{"tx_isolation": "'READ-COMMITTED'"}, + expected: "select :__vttx_isolation as `@@tx_isolation` from dual", + }, { + in: "SELECT @@transaction_isolation", + sysVar: map[string]string{"transaction_isolation": "'READ-COMMITTED'"}, + expected: "select :__vttransaction_isolation as `@@transaction_isolation` from dual", + }, { + in: "SELECT @@session.transaction_isolation", + sysVar: map[string]string{"transaction_isolation": "'READ-COMMITTED'"}, + expected: "select :__vttransaction_isolation as `@@session.transaction_isolation` from dual", + }} + + parser := NewTestParser() + for _, tc := range tests { + in.Run(tc.in, func(t *testing.T) { + require := require.New(t) + stmt, err := parser.Parse(tc.in) + require.NoError(err) + vars := NewReservedVars("v", nil) + result, err := PrepareAST( + stmt, + vars, + map[string]*querypb.BindVariable{}, + false, + "ks", + 0, + "", + tc.sysVar, + nil, + &fakeViews{}, + ) + + require.NoError(err) + + expected, err := parser.Parse(tc.expected) + require.NoError(err, "test expectation does not parse [%s]", tc.expected) + + assert.Equal(t, String(expected), String(result.AST)) + }) + } +} + +func TestRewritesWithDefaultKeyspace(in *testing.T) { + tests := []myTestCase{{ + in: "SELECT 1 from x.test", + expected: "SELECT 1 from x.test", // no change + }, { + in: "SELECT x.col as c from x.test", + expected: "SELECT x.col as c from x.test", // no change + }, { + in: "SELECT 1 from test", + expected: "SELECT 1 from sys.test", + }, { + in: "SELECT 1 from test as t", + expected: "SELECT 1 from sys.test as t", + }, { + in: "SELECT 1 from `test 24` as t", + expected: "SELECT 1 from sys.`test 24` as t", + }, { + in: "SELECT 1, (select 1 from test) from x.y", + expected: "SELECT 1, (select 1 from sys.test) from x.y", + }, { + in: "SELECT 1 from (select 2 from test) t", + expected: "SELECT 1 from (select 2 from sys.test) t", + }, { + in: "SELECT 1 from test where exists(select 2 from test)", + expected: "SELECT 1 from sys.test where exists(select 1 from sys.test)", + }, { + in: "SELECT 1 from dual", + expected: "SELECT 1 from dual", + }, { + in: "SELECT (select 2 from dual) from DUAL", + expected: "SELECT 2 as `(select 2 from dual)` from DUAL", + }} + + parser := NewTestParser() + for _, tc := range tests { + in.Run(tc.in, func(t *testing.T) { + require := require.New(t) + stmt, err := parser.Parse(tc.in) + require.NoError(err) + vars := NewReservedVars("v", nil) + result, err := PrepareAST( + stmt, + vars, + map[string]*querypb.BindVariable{}, + false, + "sys", + 0, + "", + map[string]string{}, + nil, + &fakeViews{}, + ) + + require.NoError(err) + + expected, err := parser.Parse(tc.expected) + require.NoError(err, "test expectation does not parse [%s]", tc.expected) + + assert.Equal(t, String(expected), String(result.AST)) + }) + } +} + +func TestReservedVars(t *testing.T) { + for _, prefix := range []string{"vtg", "bv"} { + t.Run("prefix_"+prefix, func(t *testing.T) { + reserved := NewReservedVars(prefix, make(BindVars)) + for i := 1; i < 1000; i++ { + require.Equal(t, fmt.Sprintf("%s%d", prefix, i), reserved.nextUnusedVar()) + } + }) + } +} + /* Skipping ColName, TableName: BenchmarkNormalize-8 1000000 2205 ns/op 821 B/op 27 allocs/op @@ -573,7 +1157,8 @@ func BenchmarkNormalize(b *testing.B) { b.Fatal(err) } for i := 0; i < b.N; i++ { - require.NoError(b, Normalize(ast, NewReservedVars("", reservedVars), map[string]*querypb.BindVariable{})) + _, err := PrepareAST(ast, NewReservedVars("", reservedVars), map[string]*querypb.BindVariable{}, true, "ks", 0, "", map[string]string{}, nil, nil) + require.NoError(b, err) } } @@ -602,7 +1187,8 @@ func BenchmarkNormalizeTraces(b *testing.B) { for i := 0; i < b.N; i++ { for i, query := range parsed { - _ = Normalize(query, NewReservedVars("", reservedVars[i]), map[string]*querypb.BindVariable{}) + _, err := PrepareAST(query, NewReservedVars("", reservedVars[i]), map[string]*querypb.BindVariable{}, true, "ks", 0, "", map[string]string{}, nil, nil) + require.NoError(b, err) } } }) diff --git a/go/vt/sqlparser/redact_query.go b/go/vt/sqlparser/redact_query.go index e6b8c009c68..2d018d7c0eb 100644 --- a/go/vt/sqlparser/redact_query.go +++ b/go/vt/sqlparser/redact_query.go @@ -28,10 +28,10 @@ func (p *Parser) RedactSQLQuery(sql string) (string, error) { return "", err } - err = Normalize(stmt, NewReservedVars("redacted", reservedVars), bv) + out, err := PrepareAST(stmt, NewReservedVars("redacted", reservedVars), bv, true, "ks", 0, "", map[string]string{}, nil, nil) if err != nil { return "", err } - return comments.Leading + String(stmt) + comments.Trailing, nil + return comments.Leading + String(out.AST) + comments.Trailing, nil } diff --git a/go/vt/sqlparser/utils.go b/go/vt/sqlparser/utils.go index b785128917f..c56e7740fc5 100644 --- a/go/vt/sqlparser/utils.go +++ b/go/vt/sqlparser/utils.go @@ -41,11 +41,12 @@ func (p *Parser) QueryMatchesTemplates(query string, queryTemplates []string) (m if err != nil { return "", err } - err = Normalize(stmt, NewReservedVars("", reservedVars), bv) + + out, err := PrepareAST(stmt, NewReservedVars("", reservedVars), bv, true, "ks", 0, "", map[string]string{}, nil, nil) if err != nil { return "", err } - normalized := CanonicalString(stmt) + normalized := CanonicalString(out.AST) return normalized, nil } diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 904805e789b..5e7e5c2a07d 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -1860,6 +1860,30 @@ func TestPassthroughDDL(t *testing.T) { sbc2.Queries = nil } +func TestShowStatus(t *testing.T) { + executor, sbc1, _, _, ctx := createExecutorEnvWithConfig(t, createExecutorConfigWithNormalizer()) + session := &vtgatepb.Session{ + TargetString: "TestExecutor", + } + + sql1 := "show slave status" + _, err := executorExec(ctx, executor, session, sql1, nil) + require.NoError(t, err) + + sql2 := "show replica status" + _, err = executorExec(ctx, executor, session, sql2, nil) + require.NoError(t, err) + + wantQueries := []*querypb.BoundQuery{{ + Sql: sql1, + BindVariables: map[string]*querypb.BindVariable{}, + }, { + Sql: sql2, + BindVariables: map[string]*querypb.BindVariable{}, + }} + assert.Equal(t, wantQueries, sbc1.Queries) +} + func TestParseEmptyTargetSingleKeyspace(t *testing.T) { r, _, _, _, _ := createExecutorEnv(t) diff --git a/go/vt/vtgate/planbuilder/builder.go b/go/vt/vtgate/planbuilder/builder.go index 065c50a6dfa..85d9f5f94ea 100644 --- a/go/vt/vtgate/planbuilder/builder.go +++ b/go/vt/vtgate/planbuilder/builder.go @@ -73,7 +73,7 @@ func (staticConfig) DirectEnabled() bool { // TestBuilder builds a plan for a query based on the specified vschema. // This method is only used from tests func TestBuilder(query string, vschema plancontext.VSchema, keyspace string) (*engine.Plan, error) { - stmt, reserved, err := vschema.Environment().Parser().Parse2(query) + stmt, known, err := vschema.Environment().Parser().Parse2(query) if err != nil { return nil, err } @@ -93,12 +93,12 @@ func TestBuilder(query string, vschema plancontext.VSchema, keyspace string) (*e }() } } - result, err := sqlparser.RewriteAST(stmt, keyspace, sqlparser.SQLSelectLimitUnset, "", nil, vschema.GetForeignKeyChecksState(), vschema) + reservedVars := sqlparser.NewReservedVars("vtg", known) + result, err := sqlparser.PrepareAST(stmt, reservedVars, map[string]*querypb.BindVariable{}, false, keyspace, sqlparser.SQLSelectLimitUnset, "", nil, vschema.GetForeignKeyChecksState(), vschema) if err != nil { return nil, err } - reservedVars := sqlparser.NewReservedVars("vtg", reserved) return BuildFromStmt(context.Background(), query, result.AST, reservedVars, vschema, result.BindVarNeeds, staticConfig{}) } diff --git a/go/vt/vtgate/planbuilder/simplifier_test.go b/go/vt/vtgate/planbuilder/simplifier_test.go index 5aeb0565f9b..012475ba021 100644 --- a/go/vt/vtgate/planbuilder/simplifier_test.go +++ b/go/vt/vtgate/planbuilder/simplifier_test.go @@ -21,6 +21,8 @@ import ( "fmt" "testing" + querypb "vitess.io/vitess/go/vt/proto/query" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -45,8 +47,8 @@ func TestSimplifyBuggyQuery(t *testing.T) { stmt, reserved, err := sqlparser.NewTestParser().Parse2(query) require.NoError(t, err) - rewritten, _ := sqlparser.RewriteAST(sqlparser.Clone(stmt), vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) reservedVars := sqlparser.NewReservedVars("vtg", reserved) + rewritten, _ := sqlparser.PrepareAST(sqlparser.Clone(stmt), reservedVars, map[string]*querypb.BindVariable{}, false, vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) simplified := simplifier.SimplifyStatement( stmt.(sqlparser.TableStatement), @@ -69,8 +71,8 @@ func TestSimplifyPanic(t *testing.T) { stmt, reserved, err := sqlparser.NewTestParser().Parse2(query) require.NoError(t, err) - rewritten, _ := sqlparser.RewriteAST(sqlparser.Clone(stmt), vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) reservedVars := sqlparser.NewReservedVars("vtg", reserved) + rewritten, _ := sqlparser.PrepareAST(sqlparser.Clone(stmt), reservedVars, map[string]*querypb.BindVariable{}, false, vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) simplified := simplifier.SimplifyStatement( stmt.(sqlparser.TableStatement), @@ -100,12 +102,12 @@ func TestUnsupportedFile(t *testing.T) { t.Skip() return } - rewritten, err := sqlparser.RewriteAST(stmt, vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) + reservedVars := sqlparser.NewReservedVars("vtg", reserved) + rewritten, err := sqlparser.PrepareAST(stmt, reservedVars, map[string]*querypb.BindVariable{}, false, vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) if err != nil { t.Skip() } - reservedVars := sqlparser.NewReservedVars("vtg", reserved) ast := rewritten.AST origQuery := sqlparser.String(ast) stmt, _, _ = sqlparser.NewTestParser().Parse2(tcase.Query) @@ -133,7 +135,7 @@ func keepSameError(query string, reservedVars *sqlparser.ReservedVars, vschema * if err != nil { panic(err) } - rewritten, _ := sqlparser.RewriteAST(stmt, vschema.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) + rewritten, _ := sqlparser.PrepareAST(stmt, reservedVars, map[string]*querypb.BindVariable{}, false, vschema.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) ast := rewritten.AST _, expected := BuildFromStmt(context.Background(), query, ast, reservedVars, vschema, rewritten.BindVarNeeds, staticConfig{}) if expected == nil { diff --git a/go/vt/vtgate/semantics/typer_test.go b/go/vt/vtgate/semantics/typer_test.go index 7de5ecf1340..1ec642b8168 100644 --- a/go/vt/vtgate/semantics/typer_test.go +++ b/go/vt/vtgate/semantics/typer_test.go @@ -41,15 +41,16 @@ func TestNormalizerAndSemanticAnalysisIntegration(t *testing.T) { for _, test := range tests { t.Run(test.query, func(t *testing.T) { - parse, err := sqlparser.NewTestParser().Parse(test.query) + parse, known, err := sqlparser.NewTestParser().Parse2(test.query) require.NoError(t, err) - err = sqlparser.Normalize(parse, sqlparser.NewReservedVars("bv", sqlparser.BindVars{}), map[string]*querypb.BindVariable{}) + rv := sqlparser.NewReservedVars("", known) + out, err := sqlparser.PrepareAST(parse, rv, map[string]*querypb.BindVariable{}, true, "d", 0, "", map[string]string{}, nil, nil) require.NoError(t, err) - st, err := Analyze(parse, "d", fakeSchemaInfo()) + st, err := Analyze(out.AST, "d", fakeSchemaInfo()) require.NoError(t, err) - bv := parse.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr.(*sqlparser.Argument) + bv := out.AST.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr.(*sqlparser.Argument) typ, found := st.ExprTypes[bv] require.True(t, found, "bindvar was not typed") require.Equal(t, test.typ, typ.Type().String()) @@ -68,15 +69,15 @@ func TestColumnCollations(t *testing.T) { for _, test := range tests { t.Run(test.query, func(t *testing.T) { - parse, err := sqlparser.NewTestParser().Parse(test.query) + ast, err := sqlparser.NewTestParser().Parse(test.query) require.NoError(t, err) - err = sqlparser.Normalize(parse, sqlparser.NewReservedVars("bv", sqlparser.BindVars{}), map[string]*querypb.BindVariable{}) + out, err := sqlparser.PrepareAST(ast, sqlparser.NewReservedVars("bv", sqlparser.BindVars{}), map[string]*querypb.BindVariable{}, true, "d", 0, "", map[string]string{}, nil, nil) require.NoError(t, err) - st, err := Analyze(parse, "d", fakeSchemaInfo()) + st, err := Analyze(out.AST, "d", fakeSchemaInfo()) require.NoError(t, err) - col := extract(parse.(*sqlparser.Select), 0) + col := extract(out.AST.(*sqlparser.Select), 0) typ, found := st.TypeForExpr(col) require.True(t, found, "column was not typed") From fd1186c6a92ce41c663fe3b45b58e0299e99173d Mon Sep 17 00:00:00 2001 From: Chaitanya Rangavajhala Date: Tue, 28 Jan 2025 08:48:29 -0500 Subject: [PATCH 06/68] VTAdmin to use VTGate's vexplain (#17508) Signed-off-by: c-r-dev --- go/vt/proto/vtadmin/vtadmin.pb.go | 641 +++++++++++------- go/vt/proto/vtadmin/vtadmin_grpc.pb.go | 40 ++ go/vt/proto/vtadmin/vtadmin_vtproto.pb.go | 396 +++++++++++ go/vt/vtadmin/api.go | 54 ++ go/vt/vtadmin/api_test.go | 180 +++++ go/vt/vtadmin/http/vexplain.go | 34 + go/vt/vtadmin/rbac/rbac.go | 2 + go/vt/vtadmin/vtsql/fakevtsql/conn.go | 11 + go/vt/vtadmin/vtsql/vtsql.go | 74 ++ proto/vtadmin.proto | 13 + web/vtadmin/src/api/http.ts | 16 + web/vtadmin/src/components/App.tsx | 5 + web/vtadmin/src/components/NavRail.tsx | 3 + .../src/components/routes/VExplain.tsx | 153 +++++ .../components/routes/VTExplain.module.scss | 2 +- web/vtadmin/src/hooks/api.ts | 8 + web/vtadmin/src/proto/vtadmin.d.ts | 227 +++++++ web/vtadmin/src/proto/vtadmin.js | 486 +++++++++++++ 18 files changed, 2085 insertions(+), 260 deletions(-) create mode 100644 go/vt/vtadmin/http/vexplain.go create mode 100644 web/vtadmin/src/components/routes/VExplain.tsx diff --git a/go/vt/proto/vtadmin/vtadmin.pb.go b/go/vt/proto/vtadmin/vtadmin.pb.go index 8b6a6997c8d..c086eb01443 100644 --- a/go/vt/proto/vtadmin/vtadmin.pb.go +++ b/go/vt/proto/vtadmin/vtadmin.pb.go @@ -7200,6 +7200,112 @@ func (x *VTExplainResponse) GetResponse() string { return "" } +type VExplainRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ClusterId string `protobuf:"bytes,1,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` + Keyspace string `protobuf:"bytes,2,opt,name=keyspace,proto3" json:"keyspace,omitempty"` + Sql string `protobuf:"bytes,3,opt,name=sql,proto3" json:"sql,omitempty"` +} + +func (x *VExplainRequest) Reset() { + *x = VExplainRequest{} + mi := &file_vtadmin_proto_msgTypes[126] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VExplainRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VExplainRequest) ProtoMessage() {} + +func (x *VExplainRequest) ProtoReflect() protoreflect.Message { + mi := &file_vtadmin_proto_msgTypes[126] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VExplainRequest.ProtoReflect.Descriptor instead. +func (*VExplainRequest) Descriptor() ([]byte, []int) { + return file_vtadmin_proto_rawDescGZIP(), []int{126} +} + +func (x *VExplainRequest) GetClusterId() string { + if x != nil { + return x.ClusterId + } + return "" +} + +func (x *VExplainRequest) GetKeyspace() string { + if x != nil { + return x.Keyspace + } + return "" +} + +func (x *VExplainRequest) GetSql() string { + if x != nil { + return x.Sql + } + return "" +} + +type VExplainResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Response string `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` +} + +func (x *VExplainResponse) Reset() { + *x = VExplainResponse{} + mi := &file_vtadmin_proto_msgTypes[127] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VExplainResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VExplainResponse) ProtoMessage() {} + +func (x *VExplainResponse) ProtoReflect() protoreflect.Message { + mi := &file_vtadmin_proto_msgTypes[127] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VExplainResponse.ProtoReflect.Descriptor instead. +func (*VExplainResponse) Descriptor() ([]byte, []int) { + return file_vtadmin_proto_rawDescGZIP(), []int{127} +} + +func (x *VExplainResponse) GetResponse() string { + if x != nil { + return x.Response + } + return "" +} + type Schema_ShardTableSize struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -7211,7 +7317,7 @@ type Schema_ShardTableSize struct { func (x *Schema_ShardTableSize) Reset() { *x = Schema_ShardTableSize{} - mi := &file_vtadmin_proto_msgTypes[129] + mi := &file_vtadmin_proto_msgTypes[131] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7223,7 +7329,7 @@ func (x *Schema_ShardTableSize) String() string { func (*Schema_ShardTableSize) ProtoMessage() {} func (x *Schema_ShardTableSize) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[129] + mi := &file_vtadmin_proto_msgTypes[131] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7267,7 +7373,7 @@ type Schema_TableSize struct { func (x *Schema_TableSize) Reset() { *x = Schema_TableSize{} - mi := &file_vtadmin_proto_msgTypes[130] + mi := &file_vtadmin_proto_msgTypes[132] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7279,7 +7385,7 @@ func (x *Schema_TableSize) String() string { func (*Schema_TableSize) ProtoMessage() {} func (x *Schema_TableSize) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[130] + mi := &file_vtadmin_proto_msgTypes[132] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7327,7 +7433,7 @@ type GetSchemaMigrationsRequest_ClusterRequest struct { func (x *GetSchemaMigrationsRequest_ClusterRequest) Reset() { *x = GetSchemaMigrationsRequest_ClusterRequest{} - mi := &file_vtadmin_proto_msgTypes[132] + mi := &file_vtadmin_proto_msgTypes[134] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7339,7 +7445,7 @@ func (x *GetSchemaMigrationsRequest_ClusterRequest) String() string { func (*GetSchemaMigrationsRequest_ClusterRequest) ProtoMessage() {} func (x *GetSchemaMigrationsRequest_ClusterRequest) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[132] + mi := &file_vtadmin_proto_msgTypes[134] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7386,7 +7492,7 @@ type ReloadSchemasResponse_KeyspaceResult struct { func (x *ReloadSchemasResponse_KeyspaceResult) Reset() { *x = ReloadSchemasResponse_KeyspaceResult{} - mi := &file_vtadmin_proto_msgTypes[135] + mi := &file_vtadmin_proto_msgTypes[137] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7398,7 +7504,7 @@ func (x *ReloadSchemasResponse_KeyspaceResult) String() string { func (*ReloadSchemasResponse_KeyspaceResult) ProtoMessage() {} func (x *ReloadSchemasResponse_KeyspaceResult) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[135] + mi := &file_vtadmin_proto_msgTypes[137] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7445,7 +7551,7 @@ type ReloadSchemasResponse_ShardResult struct { func (x *ReloadSchemasResponse_ShardResult) Reset() { *x = ReloadSchemasResponse_ShardResult{} - mi := &file_vtadmin_proto_msgTypes[136] + mi := &file_vtadmin_proto_msgTypes[138] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7457,7 +7563,7 @@ func (x *ReloadSchemasResponse_ShardResult) String() string { func (*ReloadSchemasResponse_ShardResult) ProtoMessage() {} func (x *ReloadSchemasResponse_ShardResult) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[136] + mi := &file_vtadmin_proto_msgTypes[138] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7505,7 +7611,7 @@ type ReloadSchemasResponse_TabletResult struct { func (x *ReloadSchemasResponse_TabletResult) Reset() { *x = ReloadSchemasResponse_TabletResult{} - mi := &file_vtadmin_proto_msgTypes[137] + mi := &file_vtadmin_proto_msgTypes[139] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7517,7 +7623,7 @@ func (x *ReloadSchemasResponse_TabletResult) String() string { func (*ReloadSchemasResponse_TabletResult) ProtoMessage() {} func (x *ReloadSchemasResponse_TabletResult) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[137] + mi := &file_vtadmin_proto_msgTypes[139] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8545,7 +8651,16 @@ var file_vtadmin_proto_rawDesc = []byte{ 0x52, 0x03, 0x73, 0x71, 0x6c, 0x22, 0x2f, 0x0a, 0x11, 0x56, 0x54, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xbd, 0x31, 0x0a, 0x07, 0x56, 0x54, 0x41, 0x64, 0x6d, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x5e, 0x0a, 0x0f, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, + 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x71, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x73, 0x71, 0x6c, 0x22, 0x2e, 0x0a, 0x10, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, + 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x80, 0x32, 0x0a, 0x07, 0x56, 0x54, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x12, 0x4c, 0x0a, 0x0b, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x1b, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, @@ -8929,22 +9044,26 @@ var file_vtadmin_proto_rawDesc = []byte{ 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x54, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x54, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1e, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, - 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x15, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, 0x72, 0x61, 0x66, - 0x66, 0x69, 0x63, 0x12, 0x25, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, 0x72, 0x61, 0x66, - 0x66, 0x69, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x76, 0x74, 0x63, - 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, - 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x26, 0x5a, 0x24, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, - 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x08, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x12, + 0x18, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, + 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x76, 0x74, 0x61, 0x64, + 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1e, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, + 0x69, 0x6e, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, + 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, + 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x12, 0x25, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, + 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, + 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, + 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x26, 0x5a, 0x24, 0x76, 0x69, 0x74, + 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, + 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, + 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -8960,7 +9079,7 @@ func file_vtadmin_proto_rawDescGZIP() []byte { } var file_vtadmin_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_vtadmin_proto_msgTypes = make([]protoimpl.MessageInfo, 139) +var file_vtadmin_proto_msgTypes = make([]protoimpl.MessageInfo, 141) var file_vtadmin_proto_goTypes = []any{ (Tablet_ServingState)(0), // 0: vtadmin.Tablet.ServingState (*Cluster)(nil), // 1: vtadmin.Cluster @@ -9089,207 +9208,209 @@ var file_vtadmin_proto_goTypes = []any{ (*VDiffShowResponse)(nil), // 124: vtadmin.VDiffShowResponse (*VTExplainRequest)(nil), // 125: vtadmin.VTExplainRequest (*VTExplainResponse)(nil), // 126: vtadmin.VTExplainResponse - nil, // 127: vtadmin.ClusterCellsAliases.AliasesEntry - nil, // 128: vtadmin.Keyspace.ShardsEntry - nil, // 129: vtadmin.Schema.TableSizesEntry - (*Schema_ShardTableSize)(nil), // 130: vtadmin.Schema.ShardTableSize - (*Schema_TableSize)(nil), // 131: vtadmin.Schema.TableSize - nil, // 132: vtadmin.Schema.TableSize.ByShardEntry - (*GetSchemaMigrationsRequest_ClusterRequest)(nil), // 133: vtadmin.GetSchemaMigrationsRequest.ClusterRequest - nil, // 134: vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry - nil, // 135: vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry - (*ReloadSchemasResponse_KeyspaceResult)(nil), // 136: vtadmin.ReloadSchemasResponse.KeyspaceResult - (*ReloadSchemasResponse_ShardResult)(nil), // 137: vtadmin.ReloadSchemasResponse.ShardResult - (*ReloadSchemasResponse_TabletResult)(nil), // 138: vtadmin.ReloadSchemasResponse.TabletResult - nil, // 139: vtadmin.VDiffShowResponse.ShardReportEntry - (*mysqlctl.BackupInfo)(nil), // 140: mysqlctl.BackupInfo - (*topodata.CellInfo)(nil), // 141: topodata.CellInfo - (*vtctldata.ShardReplicationPositionsResponse)(nil), // 142: vtctldata.ShardReplicationPositionsResponse - (*vtctldata.Keyspace)(nil), // 143: vtctldata.Keyspace - (*tabletmanagerdata.TableDefinition)(nil), // 144: tabletmanagerdata.TableDefinition - (*vtctldata.SchemaMigration)(nil), // 145: vtctldata.SchemaMigration - (*vtctldata.Shard)(nil), // 146: vtctldata.Shard - (*vschema.SrvVSchema)(nil), // 147: vschema.SrvVSchema - (*topodata.Tablet)(nil), // 148: topodata.Tablet - (*vschema.Keyspace)(nil), // 149: vschema.Keyspace - (*vtctldata.Workflow)(nil), // 150: vtctldata.Workflow - (*vtctldata.WorkflowDeleteRequest)(nil), // 151: vtctldata.WorkflowDeleteRequest - (*vtctldata.WorkflowSwitchTrafficRequest)(nil), // 152: vtctldata.WorkflowSwitchTrafficRequest - (*vtctldata.ApplySchemaRequest)(nil), // 153: vtctldata.ApplySchemaRequest - (*vtctldata.CancelSchemaMigrationRequest)(nil), // 154: vtctldata.CancelSchemaMigrationRequest - (*vtctldata.CleanupSchemaMigrationRequest)(nil), // 155: vtctldata.CleanupSchemaMigrationRequest - (*vtctldata.CompleteSchemaMigrationRequest)(nil), // 156: vtctldata.CompleteSchemaMigrationRequest - (*vtctldata.CreateKeyspaceRequest)(nil), // 157: vtctldata.CreateKeyspaceRequest - (*vtctldata.CreateShardRequest)(nil), // 158: vtctldata.CreateShardRequest - (*vtctldata.DeleteKeyspaceRequest)(nil), // 159: vtctldata.DeleteKeyspaceRequest - (*vtctldata.DeleteShardsRequest)(nil), // 160: vtctldata.DeleteShardsRequest - (*topodata.TabletAlias)(nil), // 161: topodata.TabletAlias - (*vtctldata.EmergencyReparentShardRequest)(nil), // 162: vtctldata.EmergencyReparentShardRequest - (*logutil.Event)(nil), // 163: logutil.Event - (*vtctldata.GetBackupsRequest)(nil), // 164: vtctldata.GetBackupsRequest - (*vtctldata.GetTransactionInfoRequest)(nil), // 165: vtctldata.GetTransactionInfoRequest - (*vtctldata.LaunchSchemaMigrationRequest)(nil), // 166: vtctldata.LaunchSchemaMigrationRequest - (*vtctldata.MaterializeCreateRequest)(nil), // 167: vtctldata.MaterializeCreateRequest - (*vtctldata.MoveTablesCompleteRequest)(nil), // 168: vtctldata.MoveTablesCompleteRequest - (*vtctldata.MoveTablesCreateRequest)(nil), // 169: vtctldata.MoveTablesCreateRequest - (*vtctldata.PlannedReparentShardRequest)(nil), // 170: vtctldata.PlannedReparentShardRequest - (*vtctldata.RetrySchemaMigrationRequest)(nil), // 171: vtctldata.RetrySchemaMigrationRequest - (*vtctldata.ReshardCreateRequest)(nil), // 172: vtctldata.ReshardCreateRequest - (*vtctldata.VDiffCreateRequest)(nil), // 173: vtctldata.VDiffCreateRequest - (*vtctldata.VDiffShowRequest)(nil), // 174: vtctldata.VDiffShowRequest - (*topodata.CellsAlias)(nil), // 175: topodata.CellsAlias - (*vtctldata.GetSchemaMigrationsRequest)(nil), // 176: vtctldata.GetSchemaMigrationsRequest - (*vtctldata.GetSrvKeyspacesResponse)(nil), // 177: vtctldata.GetSrvKeyspacesResponse - (*vtctldata.ApplySchemaResponse)(nil), // 178: vtctldata.ApplySchemaResponse - (*vtctldata.CancelSchemaMigrationResponse)(nil), // 179: vtctldata.CancelSchemaMigrationResponse - (*vtctldata.CleanupSchemaMigrationResponse)(nil), // 180: vtctldata.CleanupSchemaMigrationResponse - (*vtctldata.CompleteSchemaMigrationResponse)(nil), // 181: vtctldata.CompleteSchemaMigrationResponse - (*vtctldata.ConcludeTransactionResponse)(nil), // 182: vtctldata.ConcludeTransactionResponse - (*vtctldata.CreateShardResponse)(nil), // 183: vtctldata.CreateShardResponse - (*vtctldata.DeleteKeyspaceResponse)(nil), // 184: vtctldata.DeleteKeyspaceResponse - (*vtctldata.DeleteShardsResponse)(nil), // 185: vtctldata.DeleteShardsResponse - (*vtctldata.GetFullStatusResponse)(nil), // 186: vtctldata.GetFullStatusResponse - (*vtctldata.GetTopologyPathResponse)(nil), // 187: vtctldata.GetTopologyPathResponse - (*vtctldata.GetTransactionInfoResponse)(nil), // 188: vtctldata.GetTransactionInfoResponse - (*vtctldata.GetUnresolvedTransactionsResponse)(nil), // 189: vtctldata.GetUnresolvedTransactionsResponse - (*vtctldata.WorkflowStatusResponse)(nil), // 190: vtctldata.WorkflowStatusResponse - (*vtctldata.WorkflowUpdateResponse)(nil), // 191: vtctldata.WorkflowUpdateResponse - (*vtctldata.LaunchSchemaMigrationResponse)(nil), // 192: vtctldata.LaunchSchemaMigrationResponse - (*vtctldata.MoveTablesCompleteResponse)(nil), // 193: vtctldata.MoveTablesCompleteResponse - (*vtctldata.MaterializeCreateResponse)(nil), // 194: vtctldata.MaterializeCreateResponse - (*vtctldata.RetrySchemaMigrationResponse)(nil), // 195: vtctldata.RetrySchemaMigrationResponse - (*vtctldata.ValidateResponse)(nil), // 196: vtctldata.ValidateResponse - (*vtctldata.ValidateKeyspaceResponse)(nil), // 197: vtctldata.ValidateKeyspaceResponse - (*vtctldata.ValidateSchemaKeyspaceResponse)(nil), // 198: vtctldata.ValidateSchemaKeyspaceResponse - (*vtctldata.ValidateShardResponse)(nil), // 199: vtctldata.ValidateShardResponse - (*vtctldata.ValidateVersionKeyspaceResponse)(nil), // 200: vtctldata.ValidateVersionKeyspaceResponse - (*vtctldata.ValidateVersionShardResponse)(nil), // 201: vtctldata.ValidateVersionShardResponse - (*vtctldata.VDiffCreateResponse)(nil), // 202: vtctldata.VDiffCreateResponse - (*vtctldata.WorkflowDeleteResponse)(nil), // 203: vtctldata.WorkflowDeleteResponse - (*vtctldata.WorkflowSwitchTrafficResponse)(nil), // 204: vtctldata.WorkflowSwitchTrafficResponse + (*VExplainRequest)(nil), // 127: vtadmin.VExplainRequest + (*VExplainResponse)(nil), // 128: vtadmin.VExplainResponse + nil, // 129: vtadmin.ClusterCellsAliases.AliasesEntry + nil, // 130: vtadmin.Keyspace.ShardsEntry + nil, // 131: vtadmin.Schema.TableSizesEntry + (*Schema_ShardTableSize)(nil), // 132: vtadmin.Schema.ShardTableSize + (*Schema_TableSize)(nil), // 133: vtadmin.Schema.TableSize + nil, // 134: vtadmin.Schema.TableSize.ByShardEntry + (*GetSchemaMigrationsRequest_ClusterRequest)(nil), // 135: vtadmin.GetSchemaMigrationsRequest.ClusterRequest + nil, // 136: vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry + nil, // 137: vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry + (*ReloadSchemasResponse_KeyspaceResult)(nil), // 138: vtadmin.ReloadSchemasResponse.KeyspaceResult + (*ReloadSchemasResponse_ShardResult)(nil), // 139: vtadmin.ReloadSchemasResponse.ShardResult + (*ReloadSchemasResponse_TabletResult)(nil), // 140: vtadmin.ReloadSchemasResponse.TabletResult + nil, // 141: vtadmin.VDiffShowResponse.ShardReportEntry + (*mysqlctl.BackupInfo)(nil), // 142: mysqlctl.BackupInfo + (*topodata.CellInfo)(nil), // 143: topodata.CellInfo + (*vtctldata.ShardReplicationPositionsResponse)(nil), // 144: vtctldata.ShardReplicationPositionsResponse + (*vtctldata.Keyspace)(nil), // 145: vtctldata.Keyspace + (*tabletmanagerdata.TableDefinition)(nil), // 146: tabletmanagerdata.TableDefinition + (*vtctldata.SchemaMigration)(nil), // 147: vtctldata.SchemaMigration + (*vtctldata.Shard)(nil), // 148: vtctldata.Shard + (*vschema.SrvVSchema)(nil), // 149: vschema.SrvVSchema + (*topodata.Tablet)(nil), // 150: topodata.Tablet + (*vschema.Keyspace)(nil), // 151: vschema.Keyspace + (*vtctldata.Workflow)(nil), // 152: vtctldata.Workflow + (*vtctldata.WorkflowDeleteRequest)(nil), // 153: vtctldata.WorkflowDeleteRequest + (*vtctldata.WorkflowSwitchTrafficRequest)(nil), // 154: vtctldata.WorkflowSwitchTrafficRequest + (*vtctldata.ApplySchemaRequest)(nil), // 155: vtctldata.ApplySchemaRequest + (*vtctldata.CancelSchemaMigrationRequest)(nil), // 156: vtctldata.CancelSchemaMigrationRequest + (*vtctldata.CleanupSchemaMigrationRequest)(nil), // 157: vtctldata.CleanupSchemaMigrationRequest + (*vtctldata.CompleteSchemaMigrationRequest)(nil), // 158: vtctldata.CompleteSchemaMigrationRequest + (*vtctldata.CreateKeyspaceRequest)(nil), // 159: vtctldata.CreateKeyspaceRequest + (*vtctldata.CreateShardRequest)(nil), // 160: vtctldata.CreateShardRequest + (*vtctldata.DeleteKeyspaceRequest)(nil), // 161: vtctldata.DeleteKeyspaceRequest + (*vtctldata.DeleteShardsRequest)(nil), // 162: vtctldata.DeleteShardsRequest + (*topodata.TabletAlias)(nil), // 163: topodata.TabletAlias + (*vtctldata.EmergencyReparentShardRequest)(nil), // 164: vtctldata.EmergencyReparentShardRequest + (*logutil.Event)(nil), // 165: logutil.Event + (*vtctldata.GetBackupsRequest)(nil), // 166: vtctldata.GetBackupsRequest + (*vtctldata.GetTransactionInfoRequest)(nil), // 167: vtctldata.GetTransactionInfoRequest + (*vtctldata.LaunchSchemaMigrationRequest)(nil), // 168: vtctldata.LaunchSchemaMigrationRequest + (*vtctldata.MaterializeCreateRequest)(nil), // 169: vtctldata.MaterializeCreateRequest + (*vtctldata.MoveTablesCompleteRequest)(nil), // 170: vtctldata.MoveTablesCompleteRequest + (*vtctldata.MoveTablesCreateRequest)(nil), // 171: vtctldata.MoveTablesCreateRequest + (*vtctldata.PlannedReparentShardRequest)(nil), // 172: vtctldata.PlannedReparentShardRequest + (*vtctldata.RetrySchemaMigrationRequest)(nil), // 173: vtctldata.RetrySchemaMigrationRequest + (*vtctldata.ReshardCreateRequest)(nil), // 174: vtctldata.ReshardCreateRequest + (*vtctldata.VDiffCreateRequest)(nil), // 175: vtctldata.VDiffCreateRequest + (*vtctldata.VDiffShowRequest)(nil), // 176: vtctldata.VDiffShowRequest + (*topodata.CellsAlias)(nil), // 177: topodata.CellsAlias + (*vtctldata.GetSchemaMigrationsRequest)(nil), // 178: vtctldata.GetSchemaMigrationsRequest + (*vtctldata.GetSrvKeyspacesResponse)(nil), // 179: vtctldata.GetSrvKeyspacesResponse + (*vtctldata.ApplySchemaResponse)(nil), // 180: vtctldata.ApplySchemaResponse + (*vtctldata.CancelSchemaMigrationResponse)(nil), // 181: vtctldata.CancelSchemaMigrationResponse + (*vtctldata.CleanupSchemaMigrationResponse)(nil), // 182: vtctldata.CleanupSchemaMigrationResponse + (*vtctldata.CompleteSchemaMigrationResponse)(nil), // 183: vtctldata.CompleteSchemaMigrationResponse + (*vtctldata.ConcludeTransactionResponse)(nil), // 184: vtctldata.ConcludeTransactionResponse + (*vtctldata.CreateShardResponse)(nil), // 185: vtctldata.CreateShardResponse + (*vtctldata.DeleteKeyspaceResponse)(nil), // 186: vtctldata.DeleteKeyspaceResponse + (*vtctldata.DeleteShardsResponse)(nil), // 187: vtctldata.DeleteShardsResponse + (*vtctldata.GetFullStatusResponse)(nil), // 188: vtctldata.GetFullStatusResponse + (*vtctldata.GetTopologyPathResponse)(nil), // 189: vtctldata.GetTopologyPathResponse + (*vtctldata.GetTransactionInfoResponse)(nil), // 190: vtctldata.GetTransactionInfoResponse + (*vtctldata.GetUnresolvedTransactionsResponse)(nil), // 191: vtctldata.GetUnresolvedTransactionsResponse + (*vtctldata.WorkflowStatusResponse)(nil), // 192: vtctldata.WorkflowStatusResponse + (*vtctldata.WorkflowUpdateResponse)(nil), // 193: vtctldata.WorkflowUpdateResponse + (*vtctldata.LaunchSchemaMigrationResponse)(nil), // 194: vtctldata.LaunchSchemaMigrationResponse + (*vtctldata.MoveTablesCompleteResponse)(nil), // 195: vtctldata.MoveTablesCompleteResponse + (*vtctldata.MaterializeCreateResponse)(nil), // 196: vtctldata.MaterializeCreateResponse + (*vtctldata.RetrySchemaMigrationResponse)(nil), // 197: vtctldata.RetrySchemaMigrationResponse + (*vtctldata.ValidateResponse)(nil), // 198: vtctldata.ValidateResponse + (*vtctldata.ValidateKeyspaceResponse)(nil), // 199: vtctldata.ValidateKeyspaceResponse + (*vtctldata.ValidateSchemaKeyspaceResponse)(nil), // 200: vtctldata.ValidateSchemaKeyspaceResponse + (*vtctldata.ValidateShardResponse)(nil), // 201: vtctldata.ValidateShardResponse + (*vtctldata.ValidateVersionKeyspaceResponse)(nil), // 202: vtctldata.ValidateVersionKeyspaceResponse + (*vtctldata.ValidateVersionShardResponse)(nil), // 203: vtctldata.ValidateVersionShardResponse + (*vtctldata.VDiffCreateResponse)(nil), // 204: vtctldata.VDiffCreateResponse + (*vtctldata.WorkflowDeleteResponse)(nil), // 205: vtctldata.WorkflowDeleteResponse + (*vtctldata.WorkflowSwitchTrafficResponse)(nil), // 206: vtctldata.WorkflowSwitchTrafficResponse } var file_vtadmin_proto_depIdxs = []int32{ 1, // 0: vtadmin.ClusterBackup.cluster:type_name -> vtadmin.Cluster - 140, // 1: vtadmin.ClusterBackup.backup:type_name -> mysqlctl.BackupInfo + 142, // 1: vtadmin.ClusterBackup.backup:type_name -> mysqlctl.BackupInfo 1, // 2: vtadmin.ClusterCellsAliases.cluster:type_name -> vtadmin.Cluster - 127, // 3: vtadmin.ClusterCellsAliases.aliases:type_name -> vtadmin.ClusterCellsAliases.AliasesEntry + 129, // 3: vtadmin.ClusterCellsAliases.aliases:type_name -> vtadmin.ClusterCellsAliases.AliasesEntry 1, // 4: vtadmin.ClusterCellInfo.cluster:type_name -> vtadmin.Cluster - 141, // 5: vtadmin.ClusterCellInfo.cell_info:type_name -> topodata.CellInfo + 143, // 5: vtadmin.ClusterCellInfo.cell_info:type_name -> topodata.CellInfo 1, // 6: vtadmin.ClusterShardReplicationPosition.cluster:type_name -> vtadmin.Cluster - 142, // 7: vtadmin.ClusterShardReplicationPosition.position_info:type_name -> vtctldata.ShardReplicationPositionsResponse + 144, // 7: vtadmin.ClusterShardReplicationPosition.position_info:type_name -> vtctldata.ShardReplicationPositionsResponse 16, // 8: vtadmin.ClusterWorkflows.workflows:type_name -> vtadmin.Workflow 1, // 9: vtadmin.Keyspace.cluster:type_name -> vtadmin.Cluster - 143, // 10: vtadmin.Keyspace.keyspace:type_name -> vtctldata.Keyspace - 128, // 11: vtadmin.Keyspace.shards:type_name -> vtadmin.Keyspace.ShardsEntry + 145, // 10: vtadmin.Keyspace.keyspace:type_name -> vtctldata.Keyspace + 130, // 11: vtadmin.Keyspace.shards:type_name -> vtadmin.Keyspace.ShardsEntry 1, // 12: vtadmin.Schema.cluster:type_name -> vtadmin.Cluster - 144, // 13: vtadmin.Schema.table_definitions:type_name -> tabletmanagerdata.TableDefinition - 129, // 14: vtadmin.Schema.table_sizes:type_name -> vtadmin.Schema.TableSizesEntry + 146, // 13: vtadmin.Schema.table_definitions:type_name -> tabletmanagerdata.TableDefinition + 131, // 14: vtadmin.Schema.table_sizes:type_name -> vtadmin.Schema.TableSizesEntry 1, // 15: vtadmin.SchemaMigration.cluster:type_name -> vtadmin.Cluster - 145, // 16: vtadmin.SchemaMigration.schema_migration:type_name -> vtctldata.SchemaMigration + 147, // 16: vtadmin.SchemaMigration.schema_migration:type_name -> vtctldata.SchemaMigration 1, // 17: vtadmin.Shard.cluster:type_name -> vtadmin.Cluster - 146, // 18: vtadmin.Shard.shard:type_name -> vtctldata.Shard + 148, // 18: vtadmin.Shard.shard:type_name -> vtctldata.Shard 1, // 19: vtadmin.SrvVSchema.cluster:type_name -> vtadmin.Cluster - 147, // 20: vtadmin.SrvVSchema.srv_v_schema:type_name -> vschema.SrvVSchema + 149, // 20: vtadmin.SrvVSchema.srv_v_schema:type_name -> vschema.SrvVSchema 1, // 21: vtadmin.Tablet.cluster:type_name -> vtadmin.Cluster - 148, // 22: vtadmin.Tablet.tablet:type_name -> topodata.Tablet + 150, // 22: vtadmin.Tablet.tablet:type_name -> topodata.Tablet 0, // 23: vtadmin.Tablet.state:type_name -> vtadmin.Tablet.ServingState 1, // 24: vtadmin.VSchema.cluster:type_name -> vtadmin.Cluster - 149, // 25: vtadmin.VSchema.v_schema:type_name -> vschema.Keyspace + 151, // 25: vtadmin.VSchema.v_schema:type_name -> vschema.Keyspace 1, // 26: vtadmin.Vtctld.cluster:type_name -> vtadmin.Cluster 1, // 27: vtadmin.VTGate.cluster:type_name -> vtadmin.Cluster 1, // 28: vtadmin.Workflow.cluster:type_name -> vtadmin.Cluster - 150, // 29: vtadmin.Workflow.workflow:type_name -> vtctldata.Workflow - 151, // 30: vtadmin.WorkflowDeleteRequest.request:type_name -> vtctldata.WorkflowDeleteRequest - 152, // 31: vtadmin.WorkflowSwitchTrafficRequest.request:type_name -> vtctldata.WorkflowSwitchTrafficRequest - 153, // 32: vtadmin.ApplySchemaRequest.request:type_name -> vtctldata.ApplySchemaRequest - 154, // 33: vtadmin.CancelSchemaMigrationRequest.request:type_name -> vtctldata.CancelSchemaMigrationRequest - 155, // 34: vtadmin.CleanupSchemaMigrationRequest.request:type_name -> vtctldata.CleanupSchemaMigrationRequest - 156, // 35: vtadmin.CompleteSchemaMigrationRequest.request:type_name -> vtctldata.CompleteSchemaMigrationRequest - 157, // 36: vtadmin.CreateKeyspaceRequest.options:type_name -> vtctldata.CreateKeyspaceRequest + 152, // 29: vtadmin.Workflow.workflow:type_name -> vtctldata.Workflow + 153, // 30: vtadmin.WorkflowDeleteRequest.request:type_name -> vtctldata.WorkflowDeleteRequest + 154, // 31: vtadmin.WorkflowSwitchTrafficRequest.request:type_name -> vtctldata.WorkflowSwitchTrafficRequest + 155, // 32: vtadmin.ApplySchemaRequest.request:type_name -> vtctldata.ApplySchemaRequest + 156, // 33: vtadmin.CancelSchemaMigrationRequest.request:type_name -> vtctldata.CancelSchemaMigrationRequest + 157, // 34: vtadmin.CleanupSchemaMigrationRequest.request:type_name -> vtctldata.CleanupSchemaMigrationRequest + 158, // 35: vtadmin.CompleteSchemaMigrationRequest.request:type_name -> vtctldata.CompleteSchemaMigrationRequest + 159, // 36: vtadmin.CreateKeyspaceRequest.options:type_name -> vtctldata.CreateKeyspaceRequest 7, // 37: vtadmin.CreateKeyspaceResponse.keyspace:type_name -> vtadmin.Keyspace - 158, // 38: vtadmin.CreateShardRequest.options:type_name -> vtctldata.CreateShardRequest - 159, // 39: vtadmin.DeleteKeyspaceRequest.options:type_name -> vtctldata.DeleteKeyspaceRequest - 160, // 40: vtadmin.DeleteShardsRequest.options:type_name -> vtctldata.DeleteShardsRequest - 161, // 41: vtadmin.DeleteTabletRequest.alias:type_name -> topodata.TabletAlias + 160, // 38: vtadmin.CreateShardRequest.options:type_name -> vtctldata.CreateShardRequest + 161, // 39: vtadmin.DeleteKeyspaceRequest.options:type_name -> vtctldata.DeleteKeyspaceRequest + 162, // 40: vtadmin.DeleteShardsRequest.options:type_name -> vtctldata.DeleteShardsRequest + 163, // 41: vtadmin.DeleteTabletRequest.alias:type_name -> topodata.TabletAlias 1, // 42: vtadmin.DeleteTabletResponse.cluster:type_name -> vtadmin.Cluster - 162, // 43: vtadmin.EmergencyFailoverShardRequest.options:type_name -> vtctldata.EmergencyReparentShardRequest + 164, // 43: vtadmin.EmergencyFailoverShardRequest.options:type_name -> vtctldata.EmergencyReparentShardRequest 1, // 44: vtadmin.EmergencyFailoverShardResponse.cluster:type_name -> vtadmin.Cluster - 161, // 45: vtadmin.EmergencyFailoverShardResponse.promoted_primary:type_name -> topodata.TabletAlias - 163, // 46: vtadmin.EmergencyFailoverShardResponse.events:type_name -> logutil.Event + 163, // 45: vtadmin.EmergencyFailoverShardResponse.promoted_primary:type_name -> topodata.TabletAlias + 165, // 46: vtadmin.EmergencyFailoverShardResponse.events:type_name -> logutil.Event 61, // 47: vtadmin.FindSchemaRequest.table_size_options:type_name -> vtadmin.GetSchemaTableSizeOptions - 164, // 48: vtadmin.GetBackupsRequest.request_options:type_name -> vtctldata.GetBackupsRequest + 166, // 48: vtadmin.GetBackupsRequest.request_options:type_name -> vtctldata.GetBackupsRequest 2, // 49: vtadmin.GetBackupsResponse.backups:type_name -> vtadmin.ClusterBackup 4, // 50: vtadmin.GetCellInfosResponse.cell_infos:type_name -> vtadmin.ClusterCellInfo 3, // 51: vtadmin.GetCellsAliasesResponse.aliases:type_name -> vtadmin.ClusterCellsAliases 1, // 52: vtadmin.GetClustersResponse.clusters:type_name -> vtadmin.Cluster - 161, // 53: vtadmin.GetFullStatusRequest.alias:type_name -> topodata.TabletAlias + 163, // 53: vtadmin.GetFullStatusRequest.alias:type_name -> topodata.TabletAlias 15, // 54: vtadmin.GetGatesResponse.gates:type_name -> vtadmin.VTGate 7, // 55: vtadmin.GetKeyspacesResponse.keyspaces:type_name -> vtadmin.Keyspace 61, // 56: vtadmin.GetSchemaRequest.table_size_options:type_name -> vtadmin.GetSchemaTableSizeOptions 61, // 57: vtadmin.GetSchemasRequest.table_size_options:type_name -> vtadmin.GetSchemaTableSizeOptions 8, // 58: vtadmin.GetSchemasResponse.schemas:type_name -> vtadmin.Schema - 133, // 59: vtadmin.GetSchemaMigrationsRequest.cluster_requests:type_name -> vtadmin.GetSchemaMigrationsRequest.ClusterRequest + 135, // 59: vtadmin.GetSchemaMigrationsRequest.cluster_requests:type_name -> vtadmin.GetSchemaMigrationsRequest.ClusterRequest 9, // 60: vtadmin.GetSchemaMigrationsResponse.schema_migrations:type_name -> vtadmin.SchemaMigration 5, // 61: vtadmin.GetShardReplicationPositionsResponse.replication_positions:type_name -> vtadmin.ClusterShardReplicationPosition - 134, // 62: vtadmin.GetSrvKeyspacesResponse.srv_keyspaces:type_name -> vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry + 136, // 62: vtadmin.GetSrvKeyspacesResponse.srv_keyspaces:type_name -> vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry 11, // 63: vtadmin.GetSrvVSchemasResponse.srv_v_schemas:type_name -> vtadmin.SrvVSchema - 161, // 64: vtadmin.GetTabletRequest.alias:type_name -> topodata.TabletAlias + 163, // 64: vtadmin.GetTabletRequest.alias:type_name -> topodata.TabletAlias 12, // 65: vtadmin.GetTabletsResponse.tablets:type_name -> vtadmin.Tablet - 165, // 66: vtadmin.GetTransactionInfoRequest.request:type_name -> vtctldata.GetTransactionInfoRequest + 167, // 66: vtadmin.GetTransactionInfoRequest.request:type_name -> vtctldata.GetTransactionInfoRequest 13, // 67: vtadmin.GetVSchemasResponse.v_schemas:type_name -> vtadmin.VSchema 14, // 68: vtadmin.GetVtctldsResponse.vtctlds:type_name -> vtadmin.Vtctld - 135, // 69: vtadmin.GetWorkflowsResponse.workflows_by_cluster:type_name -> vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry - 166, // 70: vtadmin.LaunchSchemaMigrationRequest.request:type_name -> vtctldata.LaunchSchemaMigrationRequest - 167, // 71: vtadmin.MaterializeCreateRequest.request:type_name -> vtctldata.MaterializeCreateRequest - 168, // 72: vtadmin.MoveTablesCompleteRequest.request:type_name -> vtctldata.MoveTablesCompleteRequest - 169, // 73: vtadmin.MoveTablesCreateRequest.request:type_name -> vtctldata.MoveTablesCreateRequest - 161, // 74: vtadmin.PingTabletRequest.alias:type_name -> topodata.TabletAlias + 137, // 69: vtadmin.GetWorkflowsResponse.workflows_by_cluster:type_name -> vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry + 168, // 70: vtadmin.LaunchSchemaMigrationRequest.request:type_name -> vtctldata.LaunchSchemaMigrationRequest + 169, // 71: vtadmin.MaterializeCreateRequest.request:type_name -> vtctldata.MaterializeCreateRequest + 170, // 72: vtadmin.MoveTablesCompleteRequest.request:type_name -> vtctldata.MoveTablesCompleteRequest + 171, // 73: vtadmin.MoveTablesCreateRequest.request:type_name -> vtctldata.MoveTablesCreateRequest + 163, // 74: vtadmin.PingTabletRequest.alias:type_name -> topodata.TabletAlias 1, // 75: vtadmin.PingTabletResponse.cluster:type_name -> vtadmin.Cluster - 170, // 76: vtadmin.PlannedFailoverShardRequest.options:type_name -> vtctldata.PlannedReparentShardRequest + 172, // 76: vtadmin.PlannedFailoverShardRequest.options:type_name -> vtctldata.PlannedReparentShardRequest 1, // 77: vtadmin.PlannedFailoverShardResponse.cluster:type_name -> vtadmin.Cluster - 161, // 78: vtadmin.PlannedFailoverShardResponse.promoted_primary:type_name -> topodata.TabletAlias - 163, // 79: vtadmin.PlannedFailoverShardResponse.events:type_name -> logutil.Event - 161, // 80: vtadmin.RefreshStateRequest.alias:type_name -> topodata.TabletAlias + 163, // 78: vtadmin.PlannedFailoverShardResponse.promoted_primary:type_name -> topodata.TabletAlias + 165, // 79: vtadmin.PlannedFailoverShardResponse.events:type_name -> logutil.Event + 163, // 80: vtadmin.RefreshStateRequest.alias:type_name -> topodata.TabletAlias 1, // 81: vtadmin.RefreshStateResponse.cluster:type_name -> vtadmin.Cluster - 161, // 82: vtadmin.ReloadSchemasRequest.tablets:type_name -> topodata.TabletAlias - 136, // 83: vtadmin.ReloadSchemasResponse.keyspace_results:type_name -> vtadmin.ReloadSchemasResponse.KeyspaceResult - 137, // 84: vtadmin.ReloadSchemasResponse.shard_results:type_name -> vtadmin.ReloadSchemasResponse.ShardResult - 138, // 85: vtadmin.ReloadSchemasResponse.tablet_results:type_name -> vtadmin.ReloadSchemasResponse.TabletResult - 163, // 86: vtadmin.ReloadSchemaShardResponse.events:type_name -> logutil.Event - 161, // 87: vtadmin.RefreshTabletReplicationSourceRequest.alias:type_name -> topodata.TabletAlias - 161, // 88: vtadmin.RefreshTabletReplicationSourceResponse.primary:type_name -> topodata.TabletAlias + 163, // 82: vtadmin.ReloadSchemasRequest.tablets:type_name -> topodata.TabletAlias + 138, // 83: vtadmin.ReloadSchemasResponse.keyspace_results:type_name -> vtadmin.ReloadSchemasResponse.KeyspaceResult + 139, // 84: vtadmin.ReloadSchemasResponse.shard_results:type_name -> vtadmin.ReloadSchemasResponse.ShardResult + 140, // 85: vtadmin.ReloadSchemasResponse.tablet_results:type_name -> vtadmin.ReloadSchemasResponse.TabletResult + 165, // 86: vtadmin.ReloadSchemaShardResponse.events:type_name -> logutil.Event + 163, // 87: vtadmin.RefreshTabletReplicationSourceRequest.alias:type_name -> topodata.TabletAlias + 163, // 88: vtadmin.RefreshTabletReplicationSourceResponse.primary:type_name -> topodata.TabletAlias 1, // 89: vtadmin.RefreshTabletReplicationSourceResponse.cluster:type_name -> vtadmin.Cluster - 171, // 90: vtadmin.RetrySchemaMigrationRequest.request:type_name -> vtctldata.RetrySchemaMigrationRequest - 161, // 91: vtadmin.RunHealthCheckRequest.alias:type_name -> topodata.TabletAlias + 173, // 90: vtadmin.RetrySchemaMigrationRequest.request:type_name -> vtctldata.RetrySchemaMigrationRequest + 163, // 91: vtadmin.RunHealthCheckRequest.alias:type_name -> topodata.TabletAlias 1, // 92: vtadmin.RunHealthCheckResponse.cluster:type_name -> vtadmin.Cluster - 172, // 93: vtadmin.ReshardCreateRequest.request:type_name -> vtctldata.ReshardCreateRequest - 161, // 94: vtadmin.SetReadOnlyRequest.alias:type_name -> topodata.TabletAlias - 161, // 95: vtadmin.SetReadWriteRequest.alias:type_name -> topodata.TabletAlias - 161, // 96: vtadmin.StartReplicationRequest.alias:type_name -> topodata.TabletAlias + 174, // 93: vtadmin.ReshardCreateRequest.request:type_name -> vtctldata.ReshardCreateRequest + 163, // 94: vtadmin.SetReadOnlyRequest.alias:type_name -> topodata.TabletAlias + 163, // 95: vtadmin.SetReadWriteRequest.alias:type_name -> topodata.TabletAlias + 163, // 96: vtadmin.StartReplicationRequest.alias:type_name -> topodata.TabletAlias 1, // 97: vtadmin.StartReplicationResponse.cluster:type_name -> vtadmin.Cluster - 161, // 98: vtadmin.StopReplicationRequest.alias:type_name -> topodata.TabletAlias + 163, // 98: vtadmin.StopReplicationRequest.alias:type_name -> topodata.TabletAlias 1, // 99: vtadmin.StopReplicationResponse.cluster:type_name -> vtadmin.Cluster - 161, // 100: vtadmin.TabletExternallyPromotedRequest.alias:type_name -> topodata.TabletAlias + 163, // 100: vtadmin.TabletExternallyPromotedRequest.alias:type_name -> topodata.TabletAlias 1, // 101: vtadmin.TabletExternallyPromotedResponse.cluster:type_name -> vtadmin.Cluster - 161, // 102: vtadmin.TabletExternallyPromotedResponse.new_primary:type_name -> topodata.TabletAlias - 161, // 103: vtadmin.TabletExternallyPromotedResponse.old_primary:type_name -> topodata.TabletAlias - 161, // 104: vtadmin.TabletExternallyReparentedRequest.alias:type_name -> topodata.TabletAlias - 173, // 105: vtadmin.VDiffCreateRequest.request:type_name -> vtctldata.VDiffCreateRequest - 174, // 106: vtadmin.VDiffShowRequest.request:type_name -> vtctldata.VDiffShowRequest + 163, // 102: vtadmin.TabletExternallyPromotedResponse.new_primary:type_name -> topodata.TabletAlias + 163, // 103: vtadmin.TabletExternallyPromotedResponse.old_primary:type_name -> topodata.TabletAlias + 163, // 104: vtadmin.TabletExternallyReparentedRequest.alias:type_name -> topodata.TabletAlias + 175, // 105: vtadmin.VDiffCreateRequest.request:type_name -> vtctldata.VDiffCreateRequest + 176, // 106: vtadmin.VDiffShowRequest.request:type_name -> vtctldata.VDiffShowRequest 122, // 107: vtadmin.VDiffShardReport.progress:type_name -> vtadmin.VDiffProgress - 139, // 108: vtadmin.VDiffShowResponse.shard_report:type_name -> vtadmin.VDiffShowResponse.ShardReportEntry - 175, // 109: vtadmin.ClusterCellsAliases.AliasesEntry.value:type_name -> topodata.CellsAlias - 146, // 110: vtadmin.Keyspace.ShardsEntry.value:type_name -> vtctldata.Shard - 131, // 111: vtadmin.Schema.TableSizesEntry.value:type_name -> vtadmin.Schema.TableSize - 132, // 112: vtadmin.Schema.TableSize.by_shard:type_name -> vtadmin.Schema.TableSize.ByShardEntry - 130, // 113: vtadmin.Schema.TableSize.ByShardEntry.value:type_name -> vtadmin.Schema.ShardTableSize - 176, // 114: vtadmin.GetSchemaMigrationsRequest.ClusterRequest.request:type_name -> vtctldata.GetSchemaMigrationsRequest - 177, // 115: vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry.value:type_name -> vtctldata.GetSrvKeyspacesResponse + 141, // 108: vtadmin.VDiffShowResponse.shard_report:type_name -> vtadmin.VDiffShowResponse.ShardReportEntry + 177, // 109: vtadmin.ClusterCellsAliases.AliasesEntry.value:type_name -> topodata.CellsAlias + 148, // 110: vtadmin.Keyspace.ShardsEntry.value:type_name -> vtctldata.Shard + 133, // 111: vtadmin.Schema.TableSizesEntry.value:type_name -> vtadmin.Schema.TableSize + 134, // 112: vtadmin.Schema.TableSize.by_shard:type_name -> vtadmin.Schema.TableSize.ByShardEntry + 132, // 113: vtadmin.Schema.TableSize.ByShardEntry.value:type_name -> vtadmin.Schema.ShardTableSize + 178, // 114: vtadmin.GetSchemaMigrationsRequest.ClusterRequest.request:type_name -> vtctldata.GetSchemaMigrationsRequest + 179, // 115: vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry.value:type_name -> vtctldata.GetSrvKeyspacesResponse 6, // 116: vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry.value:type_name -> vtadmin.ClusterWorkflows 7, // 117: vtadmin.ReloadSchemasResponse.KeyspaceResult.keyspace:type_name -> vtadmin.Keyspace - 163, // 118: vtadmin.ReloadSchemasResponse.KeyspaceResult.events:type_name -> logutil.Event + 165, // 118: vtadmin.ReloadSchemasResponse.KeyspaceResult.events:type_name -> logutil.Event 10, // 119: vtadmin.ReloadSchemasResponse.ShardResult.shard:type_name -> vtadmin.Shard - 163, // 120: vtadmin.ReloadSchemasResponse.ShardResult.events:type_name -> logutil.Event + 165, // 120: vtadmin.ReloadSchemasResponse.ShardResult.events:type_name -> logutil.Event 12, // 121: vtadmin.ReloadSchemasResponse.TabletResult.tablet:type_name -> vtadmin.Tablet 123, // 122: vtadmin.VDiffShowResponse.ShardReportEntry.value:type_name -> vtadmin.VDiffShardReport 19, // 123: vtadmin.VTAdmin.ApplySchema:input_type -> vtadmin.ApplySchemaRequest @@ -9362,82 +9483,84 @@ var file_vtadmin_proto_depIdxs = []int32{ 120, // 190: vtadmin.VTAdmin.VDiffCreate:input_type -> vtadmin.VDiffCreateRequest 121, // 191: vtadmin.VTAdmin.VDiffShow:input_type -> vtadmin.VDiffShowRequest 125, // 192: vtadmin.VTAdmin.VTExplain:input_type -> vtadmin.VTExplainRequest - 17, // 193: vtadmin.VTAdmin.WorkflowDelete:input_type -> vtadmin.WorkflowDeleteRequest - 18, // 194: vtadmin.VTAdmin.WorkflowSwitchTraffic:input_type -> vtadmin.WorkflowSwitchTrafficRequest - 178, // 195: vtadmin.VTAdmin.ApplySchema:output_type -> vtctldata.ApplySchemaResponse - 179, // 196: vtadmin.VTAdmin.CancelSchemaMigration:output_type -> vtctldata.CancelSchemaMigrationResponse - 180, // 197: vtadmin.VTAdmin.CleanupSchemaMigration:output_type -> vtctldata.CleanupSchemaMigrationResponse - 181, // 198: vtadmin.VTAdmin.CompleteSchemaMigration:output_type -> vtctldata.CompleteSchemaMigrationResponse - 182, // 199: vtadmin.VTAdmin.ConcludeTransaction:output_type -> vtctldata.ConcludeTransactionResponse - 25, // 200: vtadmin.VTAdmin.CreateKeyspace:output_type -> vtadmin.CreateKeyspaceResponse - 183, // 201: vtadmin.VTAdmin.CreateShard:output_type -> vtctldata.CreateShardResponse - 184, // 202: vtadmin.VTAdmin.DeleteKeyspace:output_type -> vtctldata.DeleteKeyspaceResponse - 185, // 203: vtadmin.VTAdmin.DeleteShards:output_type -> vtctldata.DeleteShardsResponse - 30, // 204: vtadmin.VTAdmin.DeleteTablet:output_type -> vtadmin.DeleteTabletResponse - 32, // 205: vtadmin.VTAdmin.EmergencyFailoverShard:output_type -> vtadmin.EmergencyFailoverShardResponse - 8, // 206: vtadmin.VTAdmin.FindSchema:output_type -> vtadmin.Schema - 35, // 207: vtadmin.VTAdmin.GetBackups:output_type -> vtadmin.GetBackupsResponse - 37, // 208: vtadmin.VTAdmin.GetCellInfos:output_type -> vtadmin.GetCellInfosResponse - 39, // 209: vtadmin.VTAdmin.GetCellsAliases:output_type -> vtadmin.GetCellsAliasesResponse - 41, // 210: vtadmin.VTAdmin.GetClusters:output_type -> vtadmin.GetClustersResponse - 186, // 211: vtadmin.VTAdmin.GetFullStatus:output_type -> vtctldata.GetFullStatusResponse - 44, // 212: vtadmin.VTAdmin.GetGates:output_type -> vtadmin.GetGatesResponse - 7, // 213: vtadmin.VTAdmin.GetKeyspace:output_type -> vtadmin.Keyspace - 47, // 214: vtadmin.VTAdmin.GetKeyspaces:output_type -> vtadmin.GetKeyspacesResponse - 8, // 215: vtadmin.VTAdmin.GetSchema:output_type -> vtadmin.Schema - 50, // 216: vtadmin.VTAdmin.GetSchemas:output_type -> vtadmin.GetSchemasResponse - 52, // 217: vtadmin.VTAdmin.GetSchemaMigrations:output_type -> vtadmin.GetSchemaMigrationsResponse - 54, // 218: vtadmin.VTAdmin.GetShardReplicationPositions:output_type -> vtadmin.GetShardReplicationPositionsResponse - 177, // 219: vtadmin.VTAdmin.GetSrvKeyspace:output_type -> vtctldata.GetSrvKeyspacesResponse - 57, // 220: vtadmin.VTAdmin.GetSrvKeyspaces:output_type -> vtadmin.GetSrvKeyspacesResponse - 11, // 221: vtadmin.VTAdmin.GetSrvVSchema:output_type -> vtadmin.SrvVSchema - 60, // 222: vtadmin.VTAdmin.GetSrvVSchemas:output_type -> vtadmin.GetSrvVSchemasResponse - 12, // 223: vtadmin.VTAdmin.GetTablet:output_type -> vtadmin.Tablet - 64, // 224: vtadmin.VTAdmin.GetTablets:output_type -> vtadmin.GetTabletsResponse - 187, // 225: vtadmin.VTAdmin.GetTopologyPath:output_type -> vtctldata.GetTopologyPathResponse - 188, // 226: vtadmin.VTAdmin.GetTransactionInfo:output_type -> vtctldata.GetTransactionInfoResponse - 189, // 227: vtadmin.VTAdmin.GetUnresolvedTransactions:output_type -> vtctldata.GetUnresolvedTransactionsResponse - 13, // 228: vtadmin.VTAdmin.GetVSchema:output_type -> vtadmin.VSchema - 70, // 229: vtadmin.VTAdmin.GetVSchemas:output_type -> vtadmin.GetVSchemasResponse - 72, // 230: vtadmin.VTAdmin.GetVtctlds:output_type -> vtadmin.GetVtctldsResponse - 16, // 231: vtadmin.VTAdmin.GetWorkflow:output_type -> vtadmin.Workflow - 78, // 232: vtadmin.VTAdmin.GetWorkflows:output_type -> vtadmin.GetWorkflowsResponse - 190, // 233: vtadmin.VTAdmin.GetWorkflowStatus:output_type -> vtctldata.WorkflowStatusResponse - 191, // 234: vtadmin.VTAdmin.StartWorkflow:output_type -> vtctldata.WorkflowUpdateResponse - 191, // 235: vtadmin.VTAdmin.StopWorkflow:output_type -> vtctldata.WorkflowUpdateResponse - 192, // 236: vtadmin.VTAdmin.LaunchSchemaMigration:output_type -> vtctldata.LaunchSchemaMigrationResponse - 193, // 237: vtadmin.VTAdmin.MoveTablesComplete:output_type -> vtctldata.MoveTablesCompleteResponse - 190, // 238: vtadmin.VTAdmin.MoveTablesCreate:output_type -> vtctldata.WorkflowStatusResponse - 194, // 239: vtadmin.VTAdmin.MaterializeCreate:output_type -> vtctldata.MaterializeCreateResponse - 84, // 240: vtadmin.VTAdmin.PingTablet:output_type -> vtadmin.PingTabletResponse - 86, // 241: vtadmin.VTAdmin.PlannedFailoverShard:output_type -> vtadmin.PlannedFailoverShardResponse - 88, // 242: vtadmin.VTAdmin.RebuildKeyspaceGraph:output_type -> vtadmin.RebuildKeyspaceGraphResponse - 90, // 243: vtadmin.VTAdmin.RefreshState:output_type -> vtadmin.RefreshStateResponse - 96, // 244: vtadmin.VTAdmin.RefreshTabletReplicationSource:output_type -> vtadmin.RefreshTabletReplicationSourceResponse - 92, // 245: vtadmin.VTAdmin.ReloadSchemas:output_type -> vtadmin.ReloadSchemasResponse - 94, // 246: vtadmin.VTAdmin.ReloadSchemaShard:output_type -> vtadmin.ReloadSchemaShardResponse - 98, // 247: vtadmin.VTAdmin.RemoveKeyspaceCell:output_type -> vtadmin.RemoveKeyspaceCellResponse - 195, // 248: vtadmin.VTAdmin.RetrySchemaMigration:output_type -> vtctldata.RetrySchemaMigrationResponse - 101, // 249: vtadmin.VTAdmin.RunHealthCheck:output_type -> vtadmin.RunHealthCheckResponse - 190, // 250: vtadmin.VTAdmin.ReshardCreate:output_type -> vtctldata.WorkflowStatusResponse - 104, // 251: vtadmin.VTAdmin.SetReadOnly:output_type -> vtadmin.SetReadOnlyResponse - 106, // 252: vtadmin.VTAdmin.SetReadWrite:output_type -> vtadmin.SetReadWriteResponse - 108, // 253: vtadmin.VTAdmin.StartReplication:output_type -> vtadmin.StartReplicationResponse - 110, // 254: vtadmin.VTAdmin.StopReplication:output_type -> vtadmin.StopReplicationResponse - 112, // 255: vtadmin.VTAdmin.TabletExternallyPromoted:output_type -> vtadmin.TabletExternallyPromotedResponse - 196, // 256: vtadmin.VTAdmin.Validate:output_type -> vtctldata.ValidateResponse - 197, // 257: vtadmin.VTAdmin.ValidateKeyspace:output_type -> vtctldata.ValidateKeyspaceResponse - 198, // 258: vtadmin.VTAdmin.ValidateSchemaKeyspace:output_type -> vtctldata.ValidateSchemaKeyspaceResponse - 199, // 259: vtadmin.VTAdmin.ValidateShard:output_type -> vtctldata.ValidateShardResponse - 200, // 260: vtadmin.VTAdmin.ValidateVersionKeyspace:output_type -> vtctldata.ValidateVersionKeyspaceResponse - 201, // 261: vtadmin.VTAdmin.ValidateVersionShard:output_type -> vtctldata.ValidateVersionShardResponse - 202, // 262: vtadmin.VTAdmin.VDiffCreate:output_type -> vtctldata.VDiffCreateResponse - 124, // 263: vtadmin.VTAdmin.VDiffShow:output_type -> vtadmin.VDiffShowResponse - 126, // 264: vtadmin.VTAdmin.VTExplain:output_type -> vtadmin.VTExplainResponse - 203, // 265: vtadmin.VTAdmin.WorkflowDelete:output_type -> vtctldata.WorkflowDeleteResponse - 204, // 266: vtadmin.VTAdmin.WorkflowSwitchTraffic:output_type -> vtctldata.WorkflowSwitchTrafficResponse - 195, // [195:267] is the sub-list for method output_type - 123, // [123:195] is the sub-list for method input_type + 127, // 193: vtadmin.VTAdmin.VExplain:input_type -> vtadmin.VExplainRequest + 17, // 194: vtadmin.VTAdmin.WorkflowDelete:input_type -> vtadmin.WorkflowDeleteRequest + 18, // 195: vtadmin.VTAdmin.WorkflowSwitchTraffic:input_type -> vtadmin.WorkflowSwitchTrafficRequest + 180, // 196: vtadmin.VTAdmin.ApplySchema:output_type -> vtctldata.ApplySchemaResponse + 181, // 197: vtadmin.VTAdmin.CancelSchemaMigration:output_type -> vtctldata.CancelSchemaMigrationResponse + 182, // 198: vtadmin.VTAdmin.CleanupSchemaMigration:output_type -> vtctldata.CleanupSchemaMigrationResponse + 183, // 199: vtadmin.VTAdmin.CompleteSchemaMigration:output_type -> vtctldata.CompleteSchemaMigrationResponse + 184, // 200: vtadmin.VTAdmin.ConcludeTransaction:output_type -> vtctldata.ConcludeTransactionResponse + 25, // 201: vtadmin.VTAdmin.CreateKeyspace:output_type -> vtadmin.CreateKeyspaceResponse + 185, // 202: vtadmin.VTAdmin.CreateShard:output_type -> vtctldata.CreateShardResponse + 186, // 203: vtadmin.VTAdmin.DeleteKeyspace:output_type -> vtctldata.DeleteKeyspaceResponse + 187, // 204: vtadmin.VTAdmin.DeleteShards:output_type -> vtctldata.DeleteShardsResponse + 30, // 205: vtadmin.VTAdmin.DeleteTablet:output_type -> vtadmin.DeleteTabletResponse + 32, // 206: vtadmin.VTAdmin.EmergencyFailoverShard:output_type -> vtadmin.EmergencyFailoverShardResponse + 8, // 207: vtadmin.VTAdmin.FindSchema:output_type -> vtadmin.Schema + 35, // 208: vtadmin.VTAdmin.GetBackups:output_type -> vtadmin.GetBackupsResponse + 37, // 209: vtadmin.VTAdmin.GetCellInfos:output_type -> vtadmin.GetCellInfosResponse + 39, // 210: vtadmin.VTAdmin.GetCellsAliases:output_type -> vtadmin.GetCellsAliasesResponse + 41, // 211: vtadmin.VTAdmin.GetClusters:output_type -> vtadmin.GetClustersResponse + 188, // 212: vtadmin.VTAdmin.GetFullStatus:output_type -> vtctldata.GetFullStatusResponse + 44, // 213: vtadmin.VTAdmin.GetGates:output_type -> vtadmin.GetGatesResponse + 7, // 214: vtadmin.VTAdmin.GetKeyspace:output_type -> vtadmin.Keyspace + 47, // 215: vtadmin.VTAdmin.GetKeyspaces:output_type -> vtadmin.GetKeyspacesResponse + 8, // 216: vtadmin.VTAdmin.GetSchema:output_type -> vtadmin.Schema + 50, // 217: vtadmin.VTAdmin.GetSchemas:output_type -> vtadmin.GetSchemasResponse + 52, // 218: vtadmin.VTAdmin.GetSchemaMigrations:output_type -> vtadmin.GetSchemaMigrationsResponse + 54, // 219: vtadmin.VTAdmin.GetShardReplicationPositions:output_type -> vtadmin.GetShardReplicationPositionsResponse + 179, // 220: vtadmin.VTAdmin.GetSrvKeyspace:output_type -> vtctldata.GetSrvKeyspacesResponse + 57, // 221: vtadmin.VTAdmin.GetSrvKeyspaces:output_type -> vtadmin.GetSrvKeyspacesResponse + 11, // 222: vtadmin.VTAdmin.GetSrvVSchema:output_type -> vtadmin.SrvVSchema + 60, // 223: vtadmin.VTAdmin.GetSrvVSchemas:output_type -> vtadmin.GetSrvVSchemasResponse + 12, // 224: vtadmin.VTAdmin.GetTablet:output_type -> vtadmin.Tablet + 64, // 225: vtadmin.VTAdmin.GetTablets:output_type -> vtadmin.GetTabletsResponse + 189, // 226: vtadmin.VTAdmin.GetTopologyPath:output_type -> vtctldata.GetTopologyPathResponse + 190, // 227: vtadmin.VTAdmin.GetTransactionInfo:output_type -> vtctldata.GetTransactionInfoResponse + 191, // 228: vtadmin.VTAdmin.GetUnresolvedTransactions:output_type -> vtctldata.GetUnresolvedTransactionsResponse + 13, // 229: vtadmin.VTAdmin.GetVSchema:output_type -> vtadmin.VSchema + 70, // 230: vtadmin.VTAdmin.GetVSchemas:output_type -> vtadmin.GetVSchemasResponse + 72, // 231: vtadmin.VTAdmin.GetVtctlds:output_type -> vtadmin.GetVtctldsResponse + 16, // 232: vtadmin.VTAdmin.GetWorkflow:output_type -> vtadmin.Workflow + 78, // 233: vtadmin.VTAdmin.GetWorkflows:output_type -> vtadmin.GetWorkflowsResponse + 192, // 234: vtadmin.VTAdmin.GetWorkflowStatus:output_type -> vtctldata.WorkflowStatusResponse + 193, // 235: vtadmin.VTAdmin.StartWorkflow:output_type -> vtctldata.WorkflowUpdateResponse + 193, // 236: vtadmin.VTAdmin.StopWorkflow:output_type -> vtctldata.WorkflowUpdateResponse + 194, // 237: vtadmin.VTAdmin.LaunchSchemaMigration:output_type -> vtctldata.LaunchSchemaMigrationResponse + 195, // 238: vtadmin.VTAdmin.MoveTablesComplete:output_type -> vtctldata.MoveTablesCompleteResponse + 192, // 239: vtadmin.VTAdmin.MoveTablesCreate:output_type -> vtctldata.WorkflowStatusResponse + 196, // 240: vtadmin.VTAdmin.MaterializeCreate:output_type -> vtctldata.MaterializeCreateResponse + 84, // 241: vtadmin.VTAdmin.PingTablet:output_type -> vtadmin.PingTabletResponse + 86, // 242: vtadmin.VTAdmin.PlannedFailoverShard:output_type -> vtadmin.PlannedFailoverShardResponse + 88, // 243: vtadmin.VTAdmin.RebuildKeyspaceGraph:output_type -> vtadmin.RebuildKeyspaceGraphResponse + 90, // 244: vtadmin.VTAdmin.RefreshState:output_type -> vtadmin.RefreshStateResponse + 96, // 245: vtadmin.VTAdmin.RefreshTabletReplicationSource:output_type -> vtadmin.RefreshTabletReplicationSourceResponse + 92, // 246: vtadmin.VTAdmin.ReloadSchemas:output_type -> vtadmin.ReloadSchemasResponse + 94, // 247: vtadmin.VTAdmin.ReloadSchemaShard:output_type -> vtadmin.ReloadSchemaShardResponse + 98, // 248: vtadmin.VTAdmin.RemoveKeyspaceCell:output_type -> vtadmin.RemoveKeyspaceCellResponse + 197, // 249: vtadmin.VTAdmin.RetrySchemaMigration:output_type -> vtctldata.RetrySchemaMigrationResponse + 101, // 250: vtadmin.VTAdmin.RunHealthCheck:output_type -> vtadmin.RunHealthCheckResponse + 192, // 251: vtadmin.VTAdmin.ReshardCreate:output_type -> vtctldata.WorkflowStatusResponse + 104, // 252: vtadmin.VTAdmin.SetReadOnly:output_type -> vtadmin.SetReadOnlyResponse + 106, // 253: vtadmin.VTAdmin.SetReadWrite:output_type -> vtadmin.SetReadWriteResponse + 108, // 254: vtadmin.VTAdmin.StartReplication:output_type -> vtadmin.StartReplicationResponse + 110, // 255: vtadmin.VTAdmin.StopReplication:output_type -> vtadmin.StopReplicationResponse + 112, // 256: vtadmin.VTAdmin.TabletExternallyPromoted:output_type -> vtadmin.TabletExternallyPromotedResponse + 198, // 257: vtadmin.VTAdmin.Validate:output_type -> vtctldata.ValidateResponse + 199, // 258: vtadmin.VTAdmin.ValidateKeyspace:output_type -> vtctldata.ValidateKeyspaceResponse + 200, // 259: vtadmin.VTAdmin.ValidateSchemaKeyspace:output_type -> vtctldata.ValidateSchemaKeyspaceResponse + 201, // 260: vtadmin.VTAdmin.ValidateShard:output_type -> vtctldata.ValidateShardResponse + 202, // 261: vtadmin.VTAdmin.ValidateVersionKeyspace:output_type -> vtctldata.ValidateVersionKeyspaceResponse + 203, // 262: vtadmin.VTAdmin.ValidateVersionShard:output_type -> vtctldata.ValidateVersionShardResponse + 204, // 263: vtadmin.VTAdmin.VDiffCreate:output_type -> vtctldata.VDiffCreateResponse + 124, // 264: vtadmin.VTAdmin.VDiffShow:output_type -> vtadmin.VDiffShowResponse + 126, // 265: vtadmin.VTAdmin.VTExplain:output_type -> vtadmin.VTExplainResponse + 128, // 266: vtadmin.VTAdmin.VExplain:output_type -> vtadmin.VExplainResponse + 205, // 267: vtadmin.VTAdmin.WorkflowDelete:output_type -> vtctldata.WorkflowDeleteResponse + 206, // 268: vtadmin.VTAdmin.WorkflowSwitchTraffic:output_type -> vtctldata.WorkflowSwitchTrafficResponse + 196, // [196:269] is the sub-list for method output_type + 123, // [123:196] is the sub-list for method input_type 123, // [123:123] is the sub-list for extension type_name 123, // [123:123] is the sub-list for extension extendee 0, // [0:123] is the sub-list for field type_name @@ -9454,7 +9577,7 @@ func file_vtadmin_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_vtadmin_proto_rawDesc, NumEnums: 1, - NumMessages: 139, + NumMessages: 141, NumExtensions: 0, NumServices: 1, }, diff --git a/go/vt/proto/vtadmin/vtadmin_grpc.pb.go b/go/vt/proto/vtadmin/vtadmin_grpc.pb.go index 89fffccd424..b1b84e6c2db 100644 --- a/go/vt/proto/vtadmin/vtadmin_grpc.pb.go +++ b/go/vt/proto/vtadmin/vtadmin_grpc.pb.go @@ -215,6 +215,9 @@ type VTAdminClient interface { // VTExplain provides information on how Vitess plans to execute a // particular query. VTExplain(ctx context.Context, in *VTExplainRequest, opts ...grpc.CallOption) (*VTExplainResponse, error) + // VExplain provides information on how Vitess plans to execute a + // particular query. + VExplain(ctx context.Context, in *VExplainRequest, opts ...grpc.CallOption) (*VExplainResponse, error) // WorkflowDelete deletes a vreplication workflow. WorkflowDelete(ctx context.Context, in *WorkflowDeleteRequest, opts ...grpc.CallOption) (*vtctldata.WorkflowDeleteResponse, error) // WorkflowSwitchTraffic switches traffic for a VReplication workflow. @@ -859,6 +862,15 @@ func (c *vTAdminClient) VTExplain(ctx context.Context, in *VTExplainRequest, opt return out, nil } +func (c *vTAdminClient) VExplain(ctx context.Context, in *VExplainRequest, opts ...grpc.CallOption) (*VExplainResponse, error) { + out := new(VExplainResponse) + err := c.cc.Invoke(ctx, "/vtadmin.VTAdmin/VExplain", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *vTAdminClient) WorkflowDelete(ctx context.Context, in *WorkflowDeleteRequest, opts ...grpc.CallOption) (*vtctldata.WorkflowDeleteResponse, error) { out := new(vtctldata.WorkflowDeleteResponse) err := c.cc.Invoke(ctx, "/vtadmin.VTAdmin/WorkflowDelete", in, out, opts...) @@ -1073,6 +1085,9 @@ type VTAdminServer interface { // VTExplain provides information on how Vitess plans to execute a // particular query. VTExplain(context.Context, *VTExplainRequest) (*VTExplainResponse, error) + // VExplain provides information on how Vitess plans to execute a + // particular query. + VExplain(context.Context, *VExplainRequest) (*VExplainResponse, error) // WorkflowDelete deletes a vreplication workflow. WorkflowDelete(context.Context, *WorkflowDeleteRequest) (*vtctldata.WorkflowDeleteResponse, error) // WorkflowSwitchTraffic switches traffic for a VReplication workflow. @@ -1294,6 +1309,9 @@ func (UnimplementedVTAdminServer) VDiffShow(context.Context, *VDiffShowRequest) func (UnimplementedVTAdminServer) VTExplain(context.Context, *VTExplainRequest) (*VTExplainResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method VTExplain not implemented") } +func (UnimplementedVTAdminServer) VExplain(context.Context, *VExplainRequest) (*VExplainResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VExplain not implemented") +} func (UnimplementedVTAdminServer) WorkflowDelete(context.Context, *WorkflowDeleteRequest) (*vtctldata.WorkflowDeleteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method WorkflowDelete not implemented") } @@ -2573,6 +2591,24 @@ func _VTAdmin_VTExplain_Handler(srv interface{}, ctx context.Context, dec func(i return interceptor(ctx, in, info, handler) } +func _VTAdmin_VExplain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VExplainRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VTAdminServer).VExplain(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vtadmin.VTAdmin/VExplain", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VTAdminServer).VExplain(ctx, req.(*VExplainRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _VTAdmin_WorkflowDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(WorkflowDeleteRequest) if err := dec(in); err != nil { @@ -2896,6 +2932,10 @@ var VTAdmin_ServiceDesc = grpc.ServiceDesc{ MethodName: "VTExplain", Handler: _VTAdmin_VTExplain_Handler, }, + { + MethodName: "VExplain", + Handler: _VTAdmin_VExplain_Handler, + }, { MethodName: "WorkflowDelete", Handler: _VTAdmin_WorkflowDelete_Handler, diff --git a/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go b/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go index 82cca2cea06..31cfd018921 100644 --- a/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go +++ b/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go @@ -2801,6 +2801,42 @@ func (m *VTExplainResponse) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *VExplainRequest) CloneVT() *VExplainRequest { + if m == nil { + return (*VExplainRequest)(nil) + } + r := new(VExplainRequest) + r.ClusterId = m.ClusterId + r.Keyspace = m.Keyspace + r.Sql = m.Sql + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *VExplainRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *VExplainResponse) CloneVT() *VExplainResponse { + if m == nil { + return (*VExplainResponse)(nil) + } + r := new(VExplainResponse) + r.Response = m.Response + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *VExplainResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (m *Cluster) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -9860,6 +9896,100 @@ func (m *VTExplainResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *VExplainRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VExplainRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *VExplainRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Sql) > 0 { + i -= len(m.Sql) + copy(dAtA[i:], m.Sql) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Sql))) + i-- + dAtA[i] = 0x1a + } + if len(m.Keyspace) > 0 { + i -= len(m.Keyspace) + copy(dAtA[i:], m.Keyspace) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Keyspace))) + i-- + dAtA[i] = 0x12 + } + if len(m.ClusterId) > 0 { + i -= len(m.ClusterId) + copy(dAtA[i:], m.ClusterId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ClusterId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *VExplainResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VExplainResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *VExplainResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Response) > 0 { + i -= len(m.Response) + copy(dAtA[i:], m.Response) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Response))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *Cluster) SizeVT() (n int) { if m == nil { return 0 @@ -12560,6 +12690,42 @@ func (m *VTExplainResponse) SizeVT() (n int) { return n } +func (m *VExplainRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.ClusterId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Keyspace) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Sql) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *VExplainResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Response) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + func (m *Cluster) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -29699,3 +29865,233 @@ func (m *VTExplainResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *VExplainRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VExplainRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VExplainRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClusterId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClusterId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Keyspace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Keyspace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Sql", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Sql = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VExplainResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VExplainResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VExplainResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Response", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Response = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/go/vt/vtadmin/api.go b/go/vt/vtadmin/api.go index a54090bb044..1182d694e19 100644 --- a/go/vt/vtadmin/api.go +++ b/go/vt/vtadmin/api.go @@ -433,6 +433,7 @@ func (api *API) Handler() http.Handler { router.HandleFunc("/vdiff/{cluster_id}/show", httpAPI.Adapt(vtadminhttp.VDiffShow)).Name("API.VDiffShow") router.HandleFunc("/vtctlds", httpAPI.Adapt(vtadminhttp.GetVtctlds)).Name("API.GetVtctlds") router.HandleFunc("/vtexplain", httpAPI.Adapt(vtadminhttp.VTExplain)).Name("API.VTExplain") + router.HandleFunc("/vexplain", httpAPI.Adapt(vtadminhttp.VExplain)).Name("API.VExplain") router.HandleFunc("/workflow/{cluster_id}/{keyspace}/{name}", httpAPI.Adapt(vtadminhttp.GetWorkflow)).Name("API.GetWorkflow") router.HandleFunc("/workflows", httpAPI.Adapt(vtadminhttp.GetWorkflows)).Name("API.GetWorkflows") router.HandleFunc("/workflow/{cluster_id}/{keyspace}/{name}/status", httpAPI.Adapt(vtadminhttp.GetWorkflowStatus)).Name("API.GetWorkflowStatus") @@ -2616,6 +2617,59 @@ func (api *API) ValidateVersionShard(ctx context.Context, req *vtadminpb.Validat return res, nil } +// VExplain is part of the vtadminpb.VTAdminServer interface. +func (api *API) VExplain(ctx context.Context, req *vtadminpb.VExplainRequest) (*vtadminpb.VExplainResponse, error) { + span, ctx := trace.NewSpan(ctx, "API.VExplain") + defer span.Finish() + + if req.ClusterId == "" { + return nil, fmt.Errorf("%w: clusterID is required", errors.ErrInvalidRequest) + } + + if req.Keyspace == "" { + return nil, fmt.Errorf("%w: keyspace name is required", errors.ErrInvalidRequest) + } + + if req.Sql == "" { + return nil, fmt.Errorf("%w: SQL query is required", errors.ErrInvalidRequest) + } + + c, err := api.getClusterForRequest(req.ClusterId) + if err != nil { + return nil, err + } + + if !api.authz.IsAuthorized(ctx, c.ID, rbac.VExplainResource, rbac.GetAction) { + return nil, nil + } + + // Parser with default options. New() itself initializes with default MySQL version. + parser, err := sqlparser.New(sqlparser.Options{ + TruncateUILen: 512, + TruncateErrLen: 0, + }) + if err != nil { + return nil, err + } + + stmt, err := parser.Parse(req.GetSql()) + if err != nil { + return nil, err + } + + if _, ok := stmt.(*sqlparser.VExplainStmt); !ok { + return nil, vterrors.VT09017("Invalid VExplain statement") + } + + response, err := c.DB.VExplain(ctx, req.GetSql(), stmt.(*sqlparser.VExplainStmt)) + + if err != nil { + return nil, err + } + + return response, nil +} + // VTExplain is part of the vtadminpb.VTAdminServer interface. func (api *API) VTExplain(ctx context.Context, req *vtadminpb.VTExplainRequest) (*vtadminpb.VTExplainResponse, error) { // TODO (andrew): https://github.com/vitessio/vitess/issues/12161. diff --git a/go/vt/vtadmin/api_test.go b/go/vt/vtadmin/api_test.go index 82c744b95db..011acdf3e59 100644 --- a/go/vt/vtadmin/api_test.go +++ b/go/vt/vtadmin/api_test.go @@ -5136,6 +5136,186 @@ func TestVTExplain(t *testing.T) { } } +func TestVExplain(t *testing.T) { + tests := []struct { + name string + keyspaces []*vtctldatapb.Keyspace + shards []*vtctldatapb.Shard + srvVSchema *vschemapb.SrvVSchema + tabletSchemas map[string]*tabletmanagerdatapb.SchemaDefinition + tablets []*vtadminpb.Tablet + req *vtadminpb.VExplainRequest + expectedError error + }{ + { + name: "returns an error if cluster unspecified in request", + req: &vtadminpb.VExplainRequest{ + Keyspace: "commerce", + Sql: "vexplain all select * from customers", + }, + expectedError: vtadminerrors.ErrInvalidRequest, + }, + { + name: "returns an error if keyspace unspecified in request", + req: &vtadminpb.VExplainRequest{ + ClusterId: "c0", + Sql: "vexplain all select * from customers", + }, + expectedError: vtadminerrors.ErrInvalidRequest, + }, + { + name: "returns an error if SQL unspecified in request", + req: &vtadminpb.VExplainRequest{ + ClusterId: "c0", + Keyspace: "commerce", + }, + expectedError: vtadminerrors.ErrInvalidRequest, + }, + { + name: "runs VExplain given a valid request in a valid topology", + keyspaces: []*vtctldatapb.Keyspace{ + { + Name: "commerce", + Keyspace: &topodatapb.Keyspace{}, + }, + }, + shards: []*vtctldatapb.Shard{ + { + Name: "-", + Keyspace: "commerce", + }, + }, + srvVSchema: &vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "commerce": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "customers": {}, + }, + }, + }, + RoutingRules: &vschemapb.RoutingRules{ + Rules: []*vschemapb.RoutingRule{}, + }, + }, + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{ + "c0_cell1-0000000100": { + DatabaseSchema: "CREATE DATABASE commerce", + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: "t1", + Schema: `CREATE TABLE customers (id int(11) not null,PRIMARY KEY (id));`, + Type: "BASE", + Columns: []string{"id"}, + DataLength: 100, + RowCount: 50, + Fields: []*querypb.Field{ + { + Name: "id", + Type: querypb.Type_INT32, + }, + }, + }, + }, + }, + }, + tablets: []*vtadminpb.Tablet{ + { + Cluster: &vtadminpb.Cluster{ + Id: "c0", + Name: "cluster0", + }, + State: vtadminpb.Tablet_SERVING, + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Uid: 100, + Cell: "c0_cell1", + }, + Hostname: "tablet-cell1-a", + Keyspace: "commerce", + Shard: "-", + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + req: &vtadminpb.VExplainRequest{ + ClusterId: "c0", + Keyspace: "commerce", + Sql: "vexplain all select * from customers", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + toposerver := memorytopo.NewServer(ctx, "c0_cell1") + + tmc := testutil.TabletManagerClient{ + GetSchemaResults: map[string]struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{}, + } + + vtctldserver := testutil.NewVtctldServerWithTabletManagerClient(t, toposerver, &tmc, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return grpcvtctldserver.NewVtctldServer(vtenv.NewTestEnv(), ts) + }) + + testutil.WithTestServer(ctx, t, vtctldserver, func(t *testing.T, vtctldClient vtctldclient.VtctldClient) { + if tt.srvVSchema != nil { + err := toposerver.UpdateSrvVSchema(ctx, "c0_cell1", tt.srvVSchema) + require.NoError(t, err) + } + testutil.AddKeyspaces(ctx, t, toposerver, tt.keyspaces...) + testutil.AddShards(ctx, t, toposerver, tt.shards...) + + for _, tablet := range tt.tablets { + testutil.AddTablet(ctx, t, toposerver, tablet.Tablet, nil) + + // Adds each SchemaDefinition to the fake TabletManagerClient, or nil + // if there are no schemas for that tablet. (All tablet aliases must + // exist in the map. Otherwise, TabletManagerClient will return an error when + // looking up the schema with tablet alias that doesn't exist.) + alias := topoproto.TabletAliasString(tablet.Tablet.Alias) + tmc.GetSchemaResults[alias] = struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{ + Schema: tt.tabletSchemas[alias], + Error: nil, + } + } + + clusters := []*cluster.Cluster{ + vtadmintestutil.BuildCluster(t, vtadmintestutil.TestClusterConfig{ + Cluster: &vtadminpb.Cluster{ + Id: "c0", + Name: "cluster0", + }, + VtctldClient: vtctldClient, + Tablets: tt.tablets, + }), + } + + api := NewAPI(vtenv.NewTestEnv(), clusters, Options{}) + resp, err := api.VExplain(ctx, tt.req) + + if tt.expectedError != nil { + assert.True(t, errors.Is(err, tt.expectedError), "expected error type %w does not match actual error type %w", err, tt.expectedError) + } else { + require.NoError(t, err) + + // We don't particularly care to test the contents of the VExplain response, + // just that it exists. + assert.NotEmpty(t, resp.Response) + } + }) + }) + } +} + type ServeHTTPVtctldResponse struct { Result ServeHTTPVtctldResult `json:"result"` Ok bool `json:"ok"` diff --git a/go/vt/vtadmin/http/vexplain.go b/go/vt/vtadmin/http/vexplain.go new file mode 100644 index 00000000000..32b705e062a --- /dev/null +++ b/go/vt/vtadmin/http/vexplain.go @@ -0,0 +1,34 @@ +/* +Copyright 2025 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +import ( + "context" + + vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" +) + +// VExplain implements the http wrapper for /vexplain?cluster_id=&keyspace=&sql= +func VExplain(ctx context.Context, r Request, api *API) *JSONResponse { + query := r.URL.Query() + res, err := api.server.VExplain(ctx, &vtadminpb.VExplainRequest{ + ClusterId: query.Get("cluster_id"), + Keyspace: query.Get("keyspace"), + Sql: query.Get("sql"), + }) + return NewJSONResponse(res, err) +} diff --git a/go/vt/vtadmin/rbac/rbac.go b/go/vt/vtadmin/rbac/rbac.go index 038db46fbd5..1184b57f7ab 100644 --- a/go/vt/vtadmin/rbac/rbac.go +++ b/go/vt/vtadmin/rbac/rbac.go @@ -136,5 +136,7 @@ const ( VTExplainResource Resource = "VTExplain" + VExplainResource Resource = "VExplain" + TabletFullStatusResource Resource = "TabletFullStatus" ) diff --git a/go/vt/vtadmin/vtsql/fakevtsql/conn.go b/go/vt/vtadmin/vtsql/fakevtsql/conn.go index af9ac44ad0d..62e3ba7277a 100644 --- a/go/vt/vtadmin/vtsql/fakevtsql/conn.go +++ b/go/vt/vtadmin/vtsql/fakevtsql/conn.go @@ -90,6 +90,17 @@ func (c *conn) QueryContext(ctx context.Context, query string, args []driver.Nam }) } + return &rows{ + cols: columns, + vals: vals, + pos: 0, + closed: false, + }, nil + case "vexplain all select * from customers": + columns := []string{"VExplain"} + vals := [][]any{} + vals = append(vals, []any{"{'Table' : 'customer, 'TestPlan' : 'TestPlan'}"}) + return &rows{ cols: columns, vals: vals, diff --git a/go/vt/vtadmin/vtsql/vtsql.go b/go/vt/vtadmin/vtsql/vtsql.go index 9f23eb70443..d923730192a 100644 --- a/go/vt/vtadmin/vtsql/vtsql.go +++ b/go/vt/vtadmin/vtsql/vtsql.go @@ -17,10 +17,13 @@ limitations under the License. package vtsql import ( + "bytes" "context" "database/sql" "fmt" + "strings" "sync" + "text/tabwriter" "time" "google.golang.org/grpc/credentials/insecure" @@ -31,6 +34,7 @@ import ( "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vitessdriver" "vitess.io/vitess/go/vt/vtadmin/cluster/resolver" "vitess.io/vitess/go/vt/vtadmin/debug" @@ -45,6 +49,9 @@ type DB interface { // ShowTablets executes `SHOW vitess_tablets` and returns the result. ShowTablets(ctx context.Context) (*sql.Rows, error) + // VExplain executes query - `vexplain [ALL|PLAN|QUERIES|TRACE|KEYS] query` and returns the results + VExplain(ctx context.Context, query string, vexplainStmt *sqlparser.VExplainStmt) (*vtadminpb.VExplainResponse, error) + // Ping behaves like (*sql.DB).Ping. Ping() error // PingContext behaves like (*sql.DB).PingContext. @@ -174,6 +181,73 @@ func (vtgate *VTGateProxy) ShowTablets(ctx context.Context) (*sql.Rows, error) { return vtgate.conn.QueryContext(vtgate.getQueryContext(ctx), "SHOW vitess_tablets") } +// VExplain is part of the DB interface. +func (vtgate *VTGateProxy) VExplain(ctx context.Context, query string, vexplainStmt *sqlparser.VExplainStmt) (*vtadminpb.VExplainResponse, error) { + span, ctx := trace.NewSpan(ctx, "VTGateProxy.VExplain") + defer span.Finish() + + vtadminproto.AnnotateClusterSpan(vtgate.cluster, span) + + rows, err := vtgate.conn.QueryContext(vtgate.getQueryContext(ctx), query) + + if err != nil { + return nil, err + } + switch vexplainStmt.Type { + case sqlparser.QueriesVExplainType: + return convertVExplainQueriesResultToString(rows) + case sqlparser.AllVExplainType, sqlparser.TraceVExplainType, sqlparser.PlanVExplainType, sqlparser.KeysVExplainType: + return convertVExplainResultToString(rows) + default: + return nil, nil + } +} + +func convertVExplainResultToString(rows *sql.Rows) (*vtadminpb.VExplainResponse, error) { + var queryPlan string + for rows.Next() { + if err := rows.Scan(&queryPlan); err != nil { + return nil, err + } + } + return &vtadminpb.VExplainResponse{ + Response: queryPlan, + }, nil +} + +func convertVExplainQueriesResultToString(rows *sql.Rows) (*vtadminpb.VExplainResponse, error) { + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 0, 0, ' ', tabwriter.AlignRight) + + sep := []byte("|") + newLine := []byte("\n") + cols, _ := rows.Columns() + + if _, err := w.Write([]byte(strings.Join(cols, string(sep)) + string(newLine))); err != nil { + return nil, err + } + + row := make([][]byte, len(cols)) + rowPtr := make([]any, len(cols)) + for i := range row { + rowPtr[i] = &row[i] + } + + for rows.Next() { + if err := rows.Scan(rowPtr...); err != nil { + return nil, err + } + if _, err := w.Write(append(bytes.Join(row, sep), newLine...)); err != nil { + return nil, err + } + } + w.Flush() + + return &vtadminpb.VExplainResponse{ + Response: buf.String(), + }, nil +} + // Ping is part of the DB interface. func (vtgate *VTGateProxy) Ping() error { return vtgate.pingContext(context.Background()) diff --git a/proto/vtadmin.proto b/proto/vtadmin.proto index 963d1fa5779..1485cb485c2 100644 --- a/proto/vtadmin.proto +++ b/proto/vtadmin.proto @@ -225,6 +225,9 @@ service VTAdmin { // VTExplain provides information on how Vitess plans to execute a // particular query. rpc VTExplain(VTExplainRequest) returns (VTExplainResponse) {}; + // VExplain provides information on how Vitess plans to execute a + // particular query. + rpc VExplain(VExplainRequest) returns (VExplainResponse) {}; // WorkflowDelete deletes a vreplication workflow. rpc WorkflowDelete(WorkflowDeleteRequest) returns (vtctldata.WorkflowDeleteResponse) {}; // WorkflowSwitchTraffic switches traffic for a VReplication workflow. @@ -1097,3 +1100,13 @@ message VTExplainRequest { message VTExplainResponse { string response = 1; } + +message VExplainRequest { + string cluster_id = 1; + string keyspace = 2; + string sql = 3; +} + +message VExplainResponse { + string response = 1; +} diff --git a/web/vtadmin/src/api/http.ts b/web/vtadmin/src/api/http.ts index 674df961ef0..846f929ec52 100644 --- a/web/vtadmin/src/api/http.ts +++ b/web/vtadmin/src/api/http.ts @@ -636,6 +636,22 @@ export const fetchVTExplain = async ({ cluster, return pb.VTExplainResponse.create(result); }; +export const fetchVExplain = async ({ cluster_id, keyspace, sql }: R) => { + // As an easy enhancement for later, we can also validate the request parameters on the front-end + // instead of defaulting to '', to save a round trip. + const req = new URLSearchParams(); + req.append('cluster_id', cluster_id || ''); + req.append('keyspace', keyspace || ''); + req.append('sql', sql || ''); + + const { result } = await vtfetch(`/api/vexplain?${req}`); + + const err = pb.VExplainResponse.verify(result); + if (err) throw Error(err); + + return pb.VExplainResponse.create(result); +}; + export interface ValidateKeyspaceParams { clusterID: string; keyspace: string; diff --git a/web/vtadmin/src/components/App.tsx b/web/vtadmin/src/components/App.tsx index fd0f772ae19..e9f4bbb1844 100644 --- a/web/vtadmin/src/components/App.tsx +++ b/web/vtadmin/src/components/App.tsx @@ -30,6 +30,7 @@ import { Stream } from './routes/stream/Stream'; import { Workflows } from './routes/Workflows'; import { Workflow } from './routes/workflow/Workflow'; import { VTExplain } from './routes/VTExplain'; +import { VExplain } from './routes/VExplain'; import { Keyspace } from './routes/keyspace/Keyspace'; import { Tablet } from './routes/tablet/Tablet'; import { Backups } from './routes/Backups'; @@ -113,6 +114,10 @@ export const App = () => { + + + + diff --git a/web/vtadmin/src/components/NavRail.tsx b/web/vtadmin/src/components/NavRail.tsx index b30cd165684..c5e0c528bdd 100644 --- a/web/vtadmin/src/components/NavRail.tsx +++ b/web/vtadmin/src/components/NavRail.tsx @@ -77,6 +77,9 @@ export const NavRail = () => {
  • +
  • + +
  • diff --git a/web/vtadmin/src/components/routes/VExplain.tsx b/web/vtadmin/src/components/routes/VExplain.tsx new file mode 100644 index 00000000000..06969e63ca3 --- /dev/null +++ b/web/vtadmin/src/components/routes/VExplain.tsx @@ -0,0 +1,153 @@ +/** + * Copyright 2025 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useMemo } from 'react'; +import { orderBy } from 'lodash-es'; + +import { vtadmin as pb } from '../../proto/vtadmin'; +import { useKeyspaces, useVExplain } from '../../hooks/api'; +import { Select } from '../inputs/Select'; +import { ContentContainer } from '../layout/ContentContainer'; +import { WorkspaceHeader } from '../layout/WorkspaceHeader'; +import { WorkspaceTitle } from '../layout/WorkspaceTitle'; +import style from './VTExplain.module.scss'; +import { Code } from '../Code'; +import { useDocumentTitle } from '../../hooks/useDocumentTitle'; +import { Label } from '../inputs/Label'; + +export const VExplain = () => { + useDocumentTitle('VExplain'); + + const { data: keyspaces = [] } = useKeyspaces(); + + const [clusterID, updateCluster] = React.useState(null); + const [keyspaceName, updateKeyspace] = React.useState(null); + const [sql, updateSQL] = React.useState(null); + const [vexplainOption, updateVExplainOption] = React.useState('ALL'); + + const fetchVExplainRequestSql = function () { + return 'VEXPLAIN ' + vexplainOption + ' ' + sql; + }; + + const selectedKeyspace = + clusterID && keyspaceName + ? keyspaces?.find((k) => k.cluster?.id === clusterID && k.keyspace?.name === keyspaceName) + : null; + + const { data, error, refetch } = useVExplain( + { cluster_id: clusterID, keyspace: keyspaceName, sql: fetchVExplainRequestSql() }, + { + // Never cache, never refetch. + cacheTime: 0, + enabled: false, + refetchOnWindowFocus: false, + retry: false, + } + ); + + const onChangeKeyspace = (selectedKeyspace: pb.Keyspace | null | undefined) => { + updateCluster(selectedKeyspace?.cluster?.id); + updateKeyspace(selectedKeyspace?.keyspace?.name); + updateSQL(null); + }; + + const onChangeSQL: React.ChangeEventHandler = (e) => { + updateSQL(e.target.value); + }; + + const onSubmit: React.FormEventHandler = (e) => { + e.preventDefault(); + refetch(); + }; + + const VEXPLAIN_OPTIONS = ['ALL', 'PLAN', 'QUERIES', 'TRACE', 'KEYS']; + + const isReadyForSubmit = useMemo(() => { + return ( + typeof keyspaceName !== 'undefined' && + keyspaceName !== null && + keyspaceName !== '' && + typeof sql !== 'undefined' && + sql !== null && + sql !== '' + ); + }, [keyspaceName, sql]); + + return ( +
    + + VExplain + + +
    +
    +
    +