added queryrouted metrics and deprecated similar old metrics

Signed-off-by: Harshit Gangal <harshit@planetscale.com>

changelog/22.0/22.0.0/summary.md

@@ -4,9 +4,11 @@
 
 - **[Major Changes](#major-changes)**
   - **[Deprecations and Deletions](#deprecations-and-deletions)**
+    - [Deprecated VTGate Metrics](#vtgate-metrics)
     - [Deprecated VTTablet Flags](#vttablet-flags)
     - [Removing gh-ost and pt-osc Online DDL strategies](#ghost-ptosc)
   - **[RPC Changes](#rpc-changes)**
+  - **[VTGate Metrics]**(#vtgate-metrics)
   - **[Prefer not promoting a replica that is currently taking a backup](#reparents-prefer-not-backing-up)**
   - **[VTOrc Config File Changes](#vtorc-config-file-changes)**
   - **[VTGate Config File Changes](#vtgate-config-file-changes)**
@@ -54,6 +56,16 @@ $ vtctldclient ApplySchema --ddl-strategy="gh-ost" ...
 $ vtctldclient ApplySchema --ddl-strategy="pt-osc" ...
 ```
 
+### <a id="vtgate-metrics"/>VTGate Metrics
+
+Added two new metrics for queries with query type, plan type and tablet type as dimension.
+1. QueryProcessed - This counts the number of query executed.
+2. QueryRouted - This counts the number of shards the query was executed on.
+
+Deprecated:
+1. QueriesProcessed
+2. QueriesRouted
+
 ### <a id="reparents-prefer-not-backing-up"/>Prefer not promoting a replica that is currently taking a backup
 
 Emergency reparents now prefer not promoting replicas that are currently taking backups with a backup engine other than

go/test/endtoend/vtgate/misc_test.go

@@ -935,49 +935,59 @@ func TestQueryProcessedMetric(t *testing.T) {
 	tcases := []struct {
 		sql    string
 		metric string
+		shards int
 	}{{
 		sql:    "select id1, id2 from t1",
 		metric: "SELECT.Scatter.PRIMARY",
+		shards: 2,
 	}, {
 		sql:    "update t1 set id2 = 2 where id1 = 1",
-		metric: "UPDATE.Passthrough.PRIMARY",
+		metric: "UPDATE.MultiShard.PRIMARY",
+		shards: 2,
 	}, {
-		sql:    "delete from t1 where id1 in (1, 2)",
+		sql:    "delete from t1 where id1 in (1)",
 		metric: "DELETE.MultiShard.PRIMARY",
+		shards: 2,
 	}, {
 		sql:    "show tables",
 		metric: "SHOW.Passthrough.PRIMARY",
+		shards: 1,
 	}, {
 		sql:    "savepoint a",
 		metric: "SAVEPOINT.Transaction.PRIMARY",
+		shards: 0,
 	}, {
 		sql:    "rollback",
 		metric: "ROLLBACK.Transaction.PRIMARY",
+		shards: 0,
 	}}
 
-	initial := getQPMetric(t)
+	initialQP := getQPMetric(t, "QueryProcessed")
+	initialQR := getQPMetric(t, "QueryRouted")
 	for _, tc := range tcases {
 		t.Run(tc.sql, func(t *testing.T) {
 			utils.Exec(t, conn, tc.sql)
-			updatedMetric := getQPMetric(t)
-			assert.EqualValues(t, 1, getValue(updatedMetric, tc.metric)-getValue(initial, tc.metric))
+			updatedQP := getQPMetric(t, "QueryProcessed")
+			updatedQR := getQPMetric(t, "QueryRouted")
+			assert.EqualValues(t, 1, getValue(updatedQP, tc.metric)-getValue(initialQP, tc.metric))
+			assert.EqualValues(t, tc.shards, getValue(updatedQR, tc.metric)-getValue(initialQR, tc.metric))
 		})
 	}
 }
 
-func getQPMetric(t *testing.T) map[string]any {
+func getQPMetric(t *testing.T, metric string) map[string]any {
 	t.Helper()
 
 	vars := clusterInstance.VtgateProcess.GetVars()
 	require.NotNil(t, vars)
 
-	qpVars, exists := vars["QueryProcessed"]
+	qpVars, exists := vars[metric]
 	if !exists {
 		return nil
 	}
 
 	qpMap, ok := qpVars.(map[string]any)
-	require.True(t, ok, "query processed vars is not a map")
+	require.True(t, ok, "query metric vars is not a map")
 
 	return qpMap
 }

go/vt/vtgate/executor.go

@@ -73,14 +73,14 @@ import (
 var (
 	defaultTabletType = topodatapb.TabletType_PRIMARY
 
-	// TODO: @rafael - These two counters should be deprecated in favor of the ByTable ones in v17+. They are kept for now for backwards compatibility.
-	queriesProcessed = stats.NewCountersWithSingleLabel("QueriesProcessed", "Queries processed at vtgate by plan type", "Plan")
-	queriesRouted    = stats.NewCountersWithSingleLabel("QueriesRouted", "Queries routed from vtgate to vttablet by plan type", "Plan")
+	// TODO: @harshit/@systay - Remove these deprecated stats once we have a replacement in a released version.
+	queriesProcessed        = stats.NewCountersWithSingleLabel("QueriesProcessed", "Deprecated: Queries processed at vtgate by plan type", "Plan")
+	queriesRouted           = stats.NewCountersWithSingleLabel("QueriesRouted", "Deprecated: Queries routed from vtgate to vttablet by plan type", "Plan")
+	queriesProcessedByTable = stats.NewCountersWithMultiLabels("QueriesProcessedByTable", "Deprecated: Queries processed at vtgate by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"})
+	queriesRoutedByTable    = stats.NewCountersWithMultiLabels("QueriesRoutedByTable", "Deprecated: Queries routed from vtgate to vttablet by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"})
 
 	queryProcessed = stats.NewCountersWithMultiLabels("QueryProcessed", "Query processed at vtgate by query, plan, and tablet type", []string{"Query", "Plan", "Tablet"})
-
-	queriesProcessedByTable = stats.NewCountersWithMultiLabels("QueriesProcessedByTable", "Queries processed at vtgate by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"})
-	queriesRoutedByTable    = stats.NewCountersWithMultiLabels("QueriesRoutedByTable", "Queries routed from vtgate to vttablet by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"})
+	queryRouted    = stats.NewCountersWithMultiLabels("QueryRouted", "Query routed from vtgate to vttablet by query, plan, and tablet type", []string{"Query", "Plan", "Tablet"})
 
 	// commitMode records the timing of the commit phase of a transaction.
 	// It also tracks between different transaction mode i.e. Single, Multi and TwoPC
@@ -376,7 +376,7 @@ func (e *Executor) StreamExecute(
 		logStats.ActiveKeyspace = vc.GetKeyspace()
 
 		e.updateQueryCounts(plan.Instructions.RouteType(), plan.Instructions.GetKeyspaceName(), plan.Instructions.GetTableName(), int64(logStats.ShardQueries))
-		e.updateQueryStats(plan.QueryType.String(), plan.Type.String(), vc.TabletType().String())
+		e.updateQueryStats(plan.QueryType.String(), plan.Type.String(), vc.TabletType().String(), int64(logStats.ShardQueries))
 
 		return err
 	}
@@ -593,7 +593,7 @@ func (e *Executor) handleBegin(ctx context.Context, vcursor *econtext.VCursorImp
 	execStart := time.Now()
 	logStats.PlanTime = execStart.Sub(logStats.StartTime)
 	e.updateQueryCounts(sqlparser.StmtBegin.String(), "", "", 0)
-	e.updateQueryStats(sqlparser.StmtBegin.String(), engine.PlanTransaction.String(), vcursor.TabletType().String())
+	e.updateQueryStats(sqlparser.StmtBegin.String(), engine.PlanTransaction.String(), vcursor.TabletType().String(), 0)
 
 	begin := stmt.(*sqlparser.Begin)
 	err := e.txConn.Begin(ctx, safeSession, begin.TxAccessModes)
@@ -606,7 +606,7 @@ func (e *Executor) handleCommit(ctx context.Context, vcursor *econtext.VCursorIm
 	logStats.PlanTime = execStart.Sub(logStats.StartTime)
 	logStats.ShardQueries = uint64(len(safeSession.ShardSessions))
 	e.updateQueryCounts(sqlparser.StmtCommit.String(), "", "", int64(logStats.ShardQueries))
-	e.updateQueryStats(sqlparser.StmtCommit.String(), engine.PlanTransaction.String(), vcursor.TabletType().String())
+	e.updateQueryStats(sqlparser.StmtCommit.String(), engine.PlanTransaction.String(), vcursor.TabletType().String(), int64(logStats.ShardQueries))
 
 	err := e.txConn.Commit(ctx, safeSession)
 	logStats.CommitTime = time.Since(execStart)
@@ -623,7 +623,7 @@ func (e *Executor) handleRollback(ctx context.Context, vcursor *econtext.VCursor
 	logStats.PlanTime = execStart.Sub(logStats.StartTime)
 	logStats.ShardQueries = uint64(len(safeSession.ShardSessions))
 	e.updateQueryCounts(sqlparser.StmtRollback.String(), "", "", int64(logStats.ShardQueries))
-	e.updateQueryStats(sqlparser.StmtRollback.String(), engine.PlanTransaction.String(), vcursor.TabletType().String())
+	e.updateQueryStats(sqlparser.StmtRollback.String(), engine.PlanTransaction.String(), vcursor.TabletType().String(), int64(logStats.ShardQueries))
 
 	err := e.txConn.Rollback(ctx, safeSession)
 	logStats.CommitTime = time.Since(execStart)
@@ -635,7 +635,7 @@ func (e *Executor) handleSavepoint(ctx context.Context, vcursor *econtext.VCurso
 	logStats.PlanTime = execStart.Sub(logStats.StartTime)
 	logStats.ShardQueries = uint64(len(safeSession.ShardSessions))
 	e.updateQueryCounts(queryType, "", "", int64(logStats.ShardQueries))
-	e.updateQueryStats(queryType, engine.PlanTransaction.String(), vcursor.TabletType().String())
+	e.updateQueryStats(queryType, engine.PlanTransaction.String(), vcursor.TabletType().String(), int64(logStats.ShardQueries))
 
 	defer func() {
 		logStats.ExecuteTime = time.Since(execStart)
@@ -698,7 +698,7 @@ func (e *Executor) handleKill(ctx context.Context, mysqlCtx vtgateservice.MySQLC
 	execStart := time.Now()
 	logStats.PlanTime = execStart.Sub(logStats.StartTime)
 	e.updateQueryCounts("Kill", "", "", 0)
-	e.updateQueryStats("Kill", engine.PlanLocal.String(), vcursor.TabletType().String())
+	e.updateQueryStats("Kill", engine.PlanLocal.String(), vcursor.TabletType().String(), 0)
 
 	defer func() {
 		logStats.ExecuteTime = time.Since(execStart)
@@ -1301,8 +1301,9 @@ func (e *Executor) updateQueryCounts(planType, keyspace, tableName string, shard
 	}
 }
 
-func (e *Executor) updateQueryStats(queryType, planType, tabletType string) {
+func (e *Executor) updateQueryStats(queryType, planType, tabletType string, shards int64) {
 	queryProcessed.Add([]string{queryType, planType, tabletType}, 1)
+	queryRouted.Add([]string{queryType, planType, tabletType}, shards)
 }
 
 // VSchemaStats returns the loaded vschema stats.

go/vt/vtgate/plan_execute.go

@@ -402,7 +402,7 @@ func (e *Executor) logExecutionEnd(logStats *logstats.LogStats, execStart time.T
 	logStats.ExecuteTime = time.Since(execStart)
 
 	e.updateQueryCounts(plan.Instructions.RouteType(), plan.Instructions.GetKeyspaceName(), plan.Instructions.GetTableName(), int64(logStats.ShardQueries))
-	e.updateQueryStats(plan.QueryType.String(), plan.Type.String(), vcursor.TabletType().String())
+	e.updateQueryStats(plan.QueryType.String(), plan.Type.String(), vcursor.TabletType().String(), int64(logStats.ShardQueries))
 
 	var errCount uint64
 	if err != nil {