From 1f0d2efafb2e4620b1e7c4a38b4162dcd5a2a898 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 11 Jan 2025 20:01:51 -0500 Subject: [PATCH] Add e2e tests and fixes from them Signed-off-by: Matt Lord --- go/test/endtoend/vreplication/config_test.go | 19 ++++++++++++++----- .../vreplication/multi_tenant_test.go | 11 +++++++++-- go/test/endtoend/vreplication/vdiff2_test.go | 5 +++++ .../tabletmanager/vdiff/table_differ.go | 2 +- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/go/test/endtoend/vreplication/config_test.go b/go/test/endtoend/vreplication/config_test.go index 3e7dd08f31c..37090b0e9e9 100644 --- a/go/test/endtoend/vreplication/config_test.go +++ b/go/test/endtoend/vreplication/config_test.go @@ -45,12 +45,20 @@ import ( // default collation as it has to work across versions and the 8.0 default does not exist in 5.7. var ( // All standard user tables should have a primary key and at least one secondary key. - customerTypes = []string{"'individual'", "'soho'", "'enterprise'"} + customerTypes = []string{"'individual'", "'soho'", "'enterprise'"} + customerTable = fmt.Sprintf(`create table customer(cid int auto_increment, name varchar(128) collate utf8mb4_bin, meta json default null, + industryCategory varchar(100) generated always as (json_extract(meta, _utf8mb4'$.industry')) virtual, typ enum(%s), + sport set('football','cricket','baseball'), ts timestamp not null default current_timestamp, bits bit(2) default b'11', date1 datetime not null default '0000-00-00 00:00:00', + date2 datetime not null default '2021-00-01 00:00:00', dec80 decimal(8,0), blb blob, primary key(cid,typ), key(name)) CHARSET=utf8mb4`, strings.Join(customerTypes, ",")) + // customerTableModifiedPK has a PK on (cid) vs (cid,typ). + customerTableModifiedPK = fmt.Sprintf(`create table customer(cid int auto_increment, name varchar(128) collate utf8mb4_bin, meta json default null, + industryCategory varchar(100) generated always as (json_extract(meta, _utf8mb4'$.industry')) virtual, typ enum(%s), + sport set('football','cricket','baseball'), ts timestamp not null default current_timestamp, bits bit(2) default b'11', date1 datetime, + date2 datetime, dec80 decimal(8,0), blb blob, primary key(cid), key(name)) CHARSET=utf8mb4`, strings.Join(customerTypes, ",")) + initialProductSchema = fmt.Sprintf(` create table product(pid int, description varbinary(128), date1 datetime not null default '0000-00-00 00:00:00', date2 datetime not null default '2021-00-01 00:00:00', primary key(pid), key(date1,date2)) CHARSET=utf8mb4; -create table customer(cid int auto_increment, name varchar(128) collate utf8mb4_bin, meta json default null, industryCategory varchar(100) generated always as (json_extract(meta, _utf8mb4'$.industry')) virtual, - typ enum(%s), sport set('football','cricket','baseball'), ts timestamp not null default current_timestamp, bits bit(2) default b'11', date1 datetime not null default '0000-00-00 00:00:00', - date2 datetime not null default '2021-00-01 00:00:00', dec80 decimal(8,0), blb blob, primary key(cid,typ), key(name)) CHARSET=utf8mb4; +%s; create table customer_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence'; create table merchant(mname varchar(128), category varchar(128), primary key(mname), key(category)) CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; create table orders(oid int, cid int, pid int, mname varchar(128), price int, qty int, total int as (qty * price), total2 int as (qty * price) stored, primary key(oid), key(pid), key(cid)) CHARSET=utf8; @@ -69,7 +77,8 @@ create table `+"`blüb_tbl`"+` (id int, val1 varchar(20), `+"`blöb1`"+` blob, create table reftable (id int, val1 varchar(20), primary key(id), key(val1)); create table loadtest (id int, name varchar(256), primary key(id), key(name)); create table nopk (name varchar(128), age int unsigned); -`, strings.Join(customerTypes, ",")) +`, customerTable) + // These should always be ignored in vreplication internalSchema = ` create table _1e275eef_3b20_11eb_a38f_04ed332e05c2_20201210204529_gho(id int, val varbinary(128), primary key(id)); diff --git a/go/test/endtoend/vreplication/multi_tenant_test.go b/go/test/endtoend/vreplication/multi_tenant_test.go index 6e73303be8a..eaab232f3d0 100644 --- a/go/test/endtoend/vreplication/multi_tenant_test.go +++ b/go/test/endtoend/vreplication/multi_tenant_test.go @@ -91,7 +91,15 @@ type multiTenantMigration struct { } const ( - mtSchema = "create table t1(id int, tenant_id int, primary key(id, tenant_id)) Engine=InnoDB" + // The source/mt schema does not have the tenant_id column in the PK as adding a + // column to a table can be done as an INSTANT operation whereas modifying a table's + // PK requires a full table rebuild. So as a practical matter in production the + // source schema will likely have the tenant_id column, but NOT have it be part of + // the PK. + mtSchema = "create table t1(id int, tenant_id int, primary key(id)) Engine=InnoDB" + // The target/st schema must have the tenant_id column in the PK and the primary + // vindex. + stSchema = "create table t1(id int, tenant_id int, primary key(id, tenant_id)) Engine=InnoDB" mtVSchema = ` { "multi_tenant_spec": { @@ -127,7 +135,6 @@ const ( } } ` - stSchema = mtSchema stVSchema = ` { "tables": { diff --git a/go/test/endtoend/vreplication/vdiff2_test.go b/go/test/endtoend/vreplication/vdiff2_test.go index 6116e26eef5..501b038f5bd 100644 --- a/go/test/endtoend/vreplication/vdiff2_test.go +++ b/go/test/endtoend/vreplication/vdiff2_test.go @@ -175,6 +175,11 @@ func TestVDiff2(t *testing.T) { require.NoError(t, err) verifyClusterHealth(t, vc) + // Pre-create the customer table on the target keyspace, with the primary key on + // (cid) vs (cid,typ) on the source. This confirms that we are able to properly + // diff the table when the source and target have a different PK definition. + execVtgateQuery(t, vtgateConn, targetKs, customerTableModifiedPK) + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Primary tablets for any new shards are added in the first cell. diff --git a/go/vt/vttablet/tabletmanager/vdiff/table_differ.go b/go/vt/vttablet/tabletmanager/vdiff/table_differ.go index 6f1bee34503..3020c9b2caa 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/table_differ.go +++ b/go/vt/vttablet/tabletmanager/vdiff/table_differ.go @@ -966,7 +966,7 @@ func (td *tableDiffer) getSourcePKCols() error { for _, pkc := range sourceTable.PrimaryKeyColumns { sourcePKColumns[pkc] = struct{}{} } - for i, pkc := range td.table.PrimaryKeyColumns { + for i, pkc := range td.table.Columns { if _, ok := sourcePKColumns[pkc]; ok { td.tablePlan.sourcePkCols = append(td.tablePlan.sourcePkCols, i) }