From 308c1e0884432be926de82a82016fd66013a9e6e Mon Sep 17 00:00:00 2001 From: Brendan Dougherty Date: Thu, 15 Aug 2024 13:30:29 -0400 Subject: [PATCH 1/3] vtcombo: close query service on drop database Signed-off-by: Brendan Dougherty --- go/vt/vtcombo/tablet_map.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/go/vt/vtcombo/tablet_map.go b/go/vt/vtcombo/tablet_map.go index 77b7446fc1d..88b09846edd 100644 --- a/go/vt/vtcombo/tablet_map.go +++ b/go/vt/vtcombo/tablet_map.go @@ -254,7 +254,11 @@ func DeleteKs( tablet.tm.Stop() tablet.tm.Close() tablet.qsc.SchemaEngine().Close() - err := ts.DeleteTablet(ctx, tablet.alias) + err := tablet.qsc.QueryService().Close(ctx) + if err != nil { + return err + } + err = ts.DeleteTablet(ctx, tablet.alias) if err != nil { return err } From ed488f83b46f03112f562849e9d2588f9d61027c Mon Sep 17 00:00:00 2001 From: Brendan Dougherty Date: Mon, 19 Aug 2024 17:54:58 -0400 Subject: [PATCH 2/3] vtcombo: add test for connection leaks on drop + create Signed-off-by: Brendan Dougherty --- .../vtcombo/recreate/recreate_test.go | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/go/test/endtoend/vtcombo/recreate/recreate_test.go b/go/test/endtoend/vtcombo/recreate/recreate_test.go index e66edb7688a..56d0c454bbf 100644 --- a/go/test/endtoend/vtcombo/recreate/recreate_test.go +++ b/go/test/endtoend/vtcombo/recreate/recreate_test.go @@ -101,6 +101,9 @@ func TestDropAndRecreateWithSameShards(t *testing.T) { cur := conn.Session(ks1+"@primary", nil) + mysqlConnCountBefore, err := getMySQLConnectionCount(ctx, cur) + require.Nil(t, err) + _, err = cur.Execute(ctx, "DROP DATABASE "+ks1, nil) require.Nil(t, err) @@ -108,6 +111,26 @@ func TestDropAndRecreateWithSameShards(t *testing.T) { require.Nil(t, err) assertTabletsPresent(t) + + mysqlConnCountAfter, err := getMySQLConnectionCount(ctx, cur) + require.Nil(t, err) + + // Assert that we're not leaking mysql connections, but allow for some wiggle room due to transient connections + assert.InDelta(t, mysqlConnCountBefore, mysqlConnCountAfter, 5, + "not within allowable delta: mysqlConnCountBefore=%d, mysqlConnCountAfter=%d", mysqlConnCountBefore, mysqlConnCountAfter) +} + +func getMySQLConnectionCount(ctx context.Context, session *vtgateconn.VTGateSession) (int, error) { + result, err := session.Execute(ctx, "SELECT COUNT(*) FROM information_schema.processlist", nil) + if err != nil { + return 0, err + } + row := result.Rows[0][0] + toInt, err := row.ToInt() + if err != nil { + return 0, err + } + return toInt, nil } func assertTabletsPresent(t *testing.T) { From c903014cd4eb630de0d22a300133f54816935079 Mon Sep 17 00:00:00 2001 From: Brendan Dougherty Date: Wed, 21 Aug 2024 11:44:27 -0400 Subject: [PATCH 3/3] vtcombo: connection leak test review feedback Signed-off-by: Brendan Dougherty --- go/test/endtoend/vtcombo/recreate/recreate_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/go/test/endtoend/vtcombo/recreate/recreate_test.go b/go/test/endtoend/vtcombo/recreate/recreate_test.go index 56d0c454bbf..15cb63c3d7d 100644 --- a/go/test/endtoend/vtcombo/recreate/recreate_test.go +++ b/go/test/endtoend/vtcombo/recreate/recreate_test.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "os/exec" + "strconv" "strings" "testing" @@ -112,6 +113,8 @@ func TestDropAndRecreateWithSameShards(t *testing.T) { assertTabletsPresent(t) + // Check the connection count after the CREATE. There will be zero connections after the DROP as the database + // no longer exists, but after it gets recreated any open pools will be able to reestablish connections. mysqlConnCountAfter, err := getMySQLConnectionCount(ctx, cur) require.Nil(t, err) @@ -121,16 +124,11 @@ func TestDropAndRecreateWithSameShards(t *testing.T) { } func getMySQLConnectionCount(ctx context.Context, session *vtgateconn.VTGateSession) (int, error) { - result, err := session.Execute(ctx, "SELECT COUNT(*) FROM information_schema.processlist", nil) + result, err := session.Execute(ctx, "select variable_value from performance_schema.global_status where variable_name='threads_connected'", nil) if err != nil { return 0, err } - row := result.Rows[0][0] - toInt, err := row.ToInt() - if err != nil { - return 0, err - } - return toInt, nil + return strconv.Atoi(result.Rows[0][0].ToString()) } func assertTabletsPresent(t *testing.T) {