From c1524cb61dd66b948b1142bc9002baafd8639433 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 26 Feb 2025 08:47:37 +0000 Subject: [PATCH 01/10] Replaced v2.23 with v2.25 --- .github/workflows/issue-tests.yml | 2 +- .github/workflows/misc-migtests.yml | 2 +- .github/workflows/mysql-migtests.yml | 2 +- .github/workflows/pg-13-migtests.yml | 2 +- .github/workflows/pg-17-migtests.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml index f4446db83f..bb02c13399 100644 --- a/.github/workflows/issue-tests.yml +++ b/.github/workflows/issue-tests.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - version: [2.23.1.0-b220, 2024.1.4.0-b108, 2024.2.1.0-b185, 2.20.9.0-b47, 2.25.0.0-b489] + version: [2.25.0.0-b489, 2024.1.4.0-b108, 2024.2.1.0-b185, 2.20.9.0-b47, 2.25.0.0-b489] env: YB_VERSION: ${{ matrix.version }} YB_CONN_STR: "postgres://yugabyte:yugabyte@127.0.0.1:5433/yugabyte" diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml index 81140bd103..3cb9c75d7d 100644 --- a/.github/workflows/misc-migtests.yml +++ b/.github/workflows/misc-migtests.yml @@ -92,7 +92,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.RAHULB_S3_SECRET_ACCESS_KEY }} if: ${{ !cancelled() }} run: | - versions=("2024.2.1.0-b185" "2.20.9.0-b47" "2024.1.4.0-b108" "2.23.1.0-b220") + versions=("2024.2.1.0-b185" "2.20.9.0-b47" "2024.1.4.0-b108" "2.25.0.0-b489") for version in "${versions[@]}"; do echo "Running tests on version $version" diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml index 7d39f8ec61..81542c8145 100644 --- a/.github/workflows/mysql-migtests.yml +++ b/.github/workflows/mysql-migtests.yml @@ -10,7 +10,7 @@ jobs: run-mysql-migration-tests: strategy: matrix: - version: [2024.2.1.0-b185, 2.20.9.0-b47, 2024.1.4.0-b108, 2.23.1.0-b220] + version: [2024.2.1.0-b185, 2.20.9.0-b47, 2024.1.4.0-b108, 2.25.0.0-b489] BETA_FAST_DATA_EXPORT: [0, 1] env: BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }} diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml index 2ae5e7ab17..8d6d8147b5 100644 --- a/.github/workflows/pg-13-migtests.yml +++ b/.github/workflows/pg-13-migtests.yml @@ -10,7 +10,7 @@ jobs: run-pg-13-migration-tests: strategy: matrix: - version: [2024.2.1.0-b185, 2.20.9.0-b47, 2024.1.4.0-b108, 2.23.1.0-b220] + version: [2024.2.1.0-b185, 2.20.9.0-b47, 2024.1.4.0-b108, 2.25.0.0-b489] BETA_FAST_DATA_EXPORT: [0, 1] test_group: - offline diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml index 2a4a0450ed..91fe0a499a 100644 --- a/.github/workflows/pg-17-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -10,7 +10,7 @@ jobs: run-pg-17-migration-tests: strategy: matrix: - version: [2024.2.1.0-b185, 2.20.9.0-b47, 2024.1.4.0-b108, 2.23.1.0-b220] + version: [2024.2.1.0-b185, 2.20.9.0-b47, 2024.1.4.0-b108, 2.25.0.0-b489] BETA_FAST_DATA_EXPORT: [0, 1] test_group: - offline From 5124c2ec0d5e6e9ccf0cbb85780bcf4e5c7f49e7 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 26 Feb 2025 09:57:52 +0000 Subject: [PATCH 02/10] Test: remove extension if target is 2.25 --- migtests/tests/pg/multiple-schemas/validate | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migtests/tests/pg/multiple-schemas/validate b/migtests/tests/pg/multiple-schemas/validate index 6222862036..3a76aec727 100755 --- a/migtests/tests/pg/multiple-schemas/validate +++ b/migtests/tests/pg/multiple-schemas/validate @@ -153,6 +153,8 @@ def migration_completed_checks(tgt): print(f"target version: {tgtVersion}") if "2.16" in tgtVersion: expectedExtensions.remove("hll") + if "2.25" in tgtVersion: + expectedExtensions.remove("timetravel") for extension in expectedExtensions: assert extension in extensions, f"expected extension is not reported: {extension}" From 8a838391a8c7e5951e2b44b48ffe74dffb4a1f1e Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 26 Feb 2025 10:19:47 +0000 Subject: [PATCH 03/10] Added validation for extra extensions if present --- migtests/tests/pg/multiple-schemas/validate | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/migtests/tests/pg/multiple-schemas/validate b/migtests/tests/pg/multiple-schemas/validate index 3a76aec727..3e16ee371f 100755 --- a/migtests/tests/pg/multiple-schemas/validate +++ b/migtests/tests/pg/multiple-schemas/validate @@ -2,6 +2,7 @@ import yb import common +import sys live_migration = 'false' def main(): @@ -157,6 +158,12 @@ def migration_completed_checks(tgt): expectedExtensions.remove("timetravel") for extension in expectedExtensions: assert extension in extensions, f"expected extension is not reported: {extension}" + + # Check for extra/unexpected extensions + extra_extensions = set(extensions) - set(expectedExtensions) + if extra_extensions: + print(f"ERROR: Unexpected extensions found: {extra_extensions}") + sys.exit(1) # Exit with error status def change_expected_values_for_live_migration(): del EXPECTED_ROW_COUNT["Mixed_Case_Table_Name_Test"] From 2bb6f10d87afefda9ca780f083c5045413628305 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 26 Feb 2025 11:12:34 +0000 Subject: [PATCH 04/10] Fix indentation --- migtests/tests/pg/multiple-schemas/validate | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/migtests/tests/pg/multiple-schemas/validate b/migtests/tests/pg/multiple-schemas/validate index 3e16ee371f..3a00f5962b 100755 --- a/migtests/tests/pg/multiple-schemas/validate +++ b/migtests/tests/pg/multiple-schemas/validate @@ -160,10 +160,10 @@ def migration_completed_checks(tgt): assert extension in extensions, f"expected extension is not reported: {extension}" # Check for extra/unexpected extensions - extra_extensions = set(extensions) - set(expectedExtensions) - if extra_extensions: - print(f"ERROR: Unexpected extensions found: {extra_extensions}") - sys.exit(1) # Exit with error status + extra_extensions = set(extensions) - set(expectedExtensions) + if extra_extensions: + print(f"ERROR: Unexpected extensions found: {extra_extensions}") + sys.exit(1) # Exit with error status def change_expected_values_for_live_migration(): del EXPECTED_ROW_COUNT["Mixed_Case_Table_Name_Test"] From 72306dc4cb5a0b16ba03f5f541e195f7497deeb5 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 26 Feb 2025 12:44:28 +0000 Subject: [PATCH 05/10] Added extensions for 2.25 --- migtests/tests/pg/multiple-schemas/validate | 1 + 1 file changed, 1 insertion(+) diff --git a/migtests/tests/pg/multiple-schemas/validate b/migtests/tests/pg/multiple-schemas/validate index 3a00f5962b..5b953075dd 100755 --- a/migtests/tests/pg/multiple-schemas/validate +++ b/migtests/tests/pg/multiple-schemas/validate @@ -156,6 +156,7 @@ def migration_completed_checks(tgt): expectedExtensions.remove("hll") if "2.25" in tgtVersion: expectedExtensions.remove("timetravel") + expectedExtensions.extend(["pg_partman", "vector", "pg_walinspect", "old_snapshot", "pgtap", "yb_ycql_utils", "pg_surgery", "yb_xcluster_ddl_replication"]) for extension in expectedExtensions: assert extension in extensions, f"expected extension is not reported: {extension}" From 6d5b55f6e2cc046f2c9d677b5fadb7c4451e9365 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 26 Feb 2025 13:06:09 +0000 Subject: [PATCH 06/10] Added extensions for 2024.2 --- migtests/tests/pg/multiple-schemas/validate | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migtests/tests/pg/multiple-schemas/validate b/migtests/tests/pg/multiple-schemas/validate index 5b953075dd..8c0b1eeee4 100755 --- a/migtests/tests/pg/multiple-schemas/validate +++ b/migtests/tests/pg/multiple-schemas/validate @@ -157,6 +157,8 @@ def migration_completed_checks(tgt): if "2.25" in tgtVersion: expectedExtensions.remove("timetravel") expectedExtensions.extend(["pg_partman", "vector", "pg_walinspect", "old_snapshot", "pgtap", "yb_ycql_utils", "pg_surgery", "yb_xcluster_ddl_replication"]) + if "2024.2" in tgtVersion: + expectedExtensions.extend(["pg_partman", "vector", "pgtap", "yb_ycql_utils", "yb_xcluster_ddl_replication"]) for extension in expectedExtensions: assert extension in extensions, f"expected extension is not reported: {extension}" From 5ea2b10eca481ef792e42a6e74efbebfda1c5833 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 26 Feb 2025 13:40:14 +0000 Subject: [PATCH 07/10] Added expected extensions for YB versions --- migtests/tests/pg/multiple-schemas/validate | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/migtests/tests/pg/multiple-schemas/validate b/migtests/tests/pg/multiple-schemas/validate index 8c0b1eeee4..0858b29064 100755 --- a/migtests/tests/pg/multiple-schemas/validate +++ b/migtests/tests/pg/multiple-schemas/validate @@ -146,19 +146,19 @@ def migration_completed_checks(tgt): "orafce", "pageinspect", "pg_buffercache", "pg_cron", "pg_freespacemap", "pg_hint_plan", "pg_prewarm", "pg_stat_monitor", "pg_stat_statements", "pg_trgm", "pg_visibility", "pgaudit", "pgcrypto", "pgrowlocks", "pgstattuple", "plpgsql", "postgres_fdw", "refint", "seg", "sslinfo", "tablefunc", "tcn", "timetravel", "tsm_system_rows", - "tsm_system_time", "unaccent", "uuid-ossp", "yb_pg_metrics", "yb_test_extension" ] + "tsm_system_time", "unaccent", "uuid-ossp", "yb_pg_metrics", "yb_test_extension", "vector" ] extensions = tgt.get_available_extensions() print(f"expected extensions: {expectedExtensions}") print(f"reported extensions: {extensions}") tgtVersion = tgt.get_target_version() print(f"target version: {tgtVersion}") - if "2.16" in tgtVersion: - expectedExtensions.remove("hll") if "2.25" in tgtVersion: expectedExtensions.remove("timetravel") - expectedExtensions.extend(["pg_partman", "vector", "pg_walinspect", "old_snapshot", "pgtap", "yb_ycql_utils", "pg_surgery", "yb_xcluster_ddl_replication"]) + expectedExtensions.extend(["pg_partman", "pg_walinspect", "old_snapshot", "pgtap", "yb_ycql_utils", "pg_surgery", "yb_xcluster_ddl_replication"]) if "2024.2" in tgtVersion: - expectedExtensions.extend(["pg_partman", "vector", "pgtap", "yb_ycql_utils", "yb_xcluster_ddl_replication"]) + expectedExtensions.extend(["pg_partman", "pgtap", "yb_ycql_utils", "yb_xcluster_ddl_replication"]) + if "2024.1" in tgtVersion: + expectedExtensions.extend(["yb_ycql_utils"]) for extension in expectedExtensions: assert extension in extensions, f"expected extension is not reported: {extension}" From ffddc021a1e99363a3cde29d0fd070340da0801b Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Mon, 3 Mar 2025 13:09:36 +0000 Subject: [PATCH 08/10] Fixed MySQL datatypes test for 2.25 --- .../tests/mysql/datatypes/mysql_data_types_automation.sql | 2 +- migtests/tests/mysql/datatypes/validate | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/migtests/tests/mysql/datatypes/mysql_data_types_automation.sql b/migtests/tests/mysql/datatypes/mysql_data_types_automation.sql index 150af21576..1ac64b37b6 100644 --- a/migtests/tests/mysql/datatypes/mysql_data_types_automation.sql +++ b/migtests/tests/mysql/datatypes/mysql_data_types_automation.sql @@ -22,7 +22,7 @@ insert into fixed_point_types values (0.987546789,0,12397.64563,1264.12); drop table if exists floating_point_types; create table if not exists floating_point_types(id int,float_type float, double_type double, real_type REAL); insert into floating_point_types values (1,0.456,0.12313,0.9); - insert into floating_point_types values (2,123456.456,0.1246533,1.9); +insert into floating_point_types values (2,123456.456,0.1246533,1.9); insert into floating_point_types values (3,0,0,0); insert into floating_point_types values (4,126.156,0.1,1.9999); diff --git a/migtests/tests/mysql/datatypes/validate b/migtests/tests/mysql/datatypes/validate index db3de06f14..f1d7e22e33 100755 --- a/migtests/tests/mysql/datatypes/validate +++ b/migtests/tests/mysql/datatypes/validate @@ -143,6 +143,11 @@ def migration_completed_checks(tgt): for table_name, row_count in EXPECTED_ROW_COUNT.items(): print(f"table_name: {table_name}, row_count: {got_row_count[table_name]}") assert row_count == got_row_count[table_name] + + tgtVersion = tgt.get_target_version() + + if "2.25" in tgtVersion: + EXPECTED_SUM_OF_COLUMN['floating_point_types']['float_type'] = '123582.61200000001' for table_name, column_names_with_sum in EXPECTED_SUM_OF_COLUMN.items(): print(f"table_name: {table_name} ---- ") From 931675dfe1276b91778e80417226db408b217840 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Mon, 3 Mar 2025 13:30:08 +0000 Subject: [PATCH 09/10] Added condition based on debezium export --- migtests/tests/mysql/datatypes/validate | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migtests/tests/mysql/datatypes/validate b/migtests/tests/mysql/datatypes/validate index f1d7e22e33..5016a8686a 100755 --- a/migtests/tests/mysql/datatypes/validate +++ b/migtests/tests/mysql/datatypes/validate @@ -145,8 +145,8 @@ def migration_completed_checks(tgt): assert row_count == got_row_count[table_name] tgtVersion = tgt.get_target_version() - - if "2.25" in tgtVersion: + + if "2.25" in tgtVersion and os.environ.get('BETA_FAST_DATA_EXPORT') == '1': EXPECTED_SUM_OF_COLUMN['floating_point_types']['float_type'] = '123582.61200000001' for table_name, column_names_with_sum in EXPECTED_SUM_OF_COLUMN.items(): From d56e11c59c773490228c399b97d5633e6b6cbdae Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Mon, 3 Mar 2025 15:28:29 +0000 Subject: [PATCH 10/10] Import os --- migtests/tests/mysql/datatypes/validate | 1 + 1 file changed, 1 insertion(+) diff --git a/migtests/tests/mysql/datatypes/validate b/migtests/tests/mysql/datatypes/validate index 5016a8686a..876f8cec8f 100755 --- a/migtests/tests/mysql/datatypes/validate +++ b/migtests/tests/mysql/datatypes/validate @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import yb +import os def main(): yb.run_checks(migration_completed_checks)