From d8fb5021d2d0ddaa94de2134e042f1d5e524e919 Mon Sep 17 00:00:00 2001 From: vamshi Date: Thu, 20 Feb 2025 14:06:23 -0800 Subject: [PATCH] Add tests for merge mode (#161) Co-authored-by: Vamshi Kolanu --- README.md | 2 +- tests/functional/adapter/iceberg_files.py | 29 +++++++++++++++++++++ tests/functional/adapter/test_basic.py | 7 +++++ tests/functional/adapter/test_iceberg_v1.py | 12 +++++++++ tests/functional/adapter/test_iceberg_v2.py | 11 ++++++++ 5 files changed, 60 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cfe89b0..383589b 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Support for [On-Schema Change](https://docs.getdbt.com/docs/build/incremental-mo |Materialization: Incremental - Append | Yes | Yes| |Materialization: Incremental - Append with Partitions | Yes | Yes| |Materialization: Incremental - Insert+Overwrite with Partitions | Yes | No | -|Materialization: Incremental - Merge | No | No | +|Materialization: Incremental - Merge | Yes | Yes | |Materialization: Ephemeral | No | No | |Seeds | Yes | Yes | |Tests | Yes | Yes | diff --git a/tests/functional/adapter/iceberg_files.py b/tests/functional/adapter/iceberg_files.py index a419481..735d9b9 100644 --- a/tests/functional/adapter/iceberg_files.py +++ b/tests/functional/adapter/iceberg_files.py @@ -34,6 +34,35 @@ + model_incremental ) +merge_iceberg_sql = """ +{{ config( + materialized = 'incremental', + unique_key = 'id', + incremental_strategy='merge', + merge_exclude_columns=['msg'], + table_type='iceberg' +) }} + +{% if not is_incremental() %} + +-- data for first invocation of model + +select 1 as id, 'hello' as msg, 'blue' as color +union all +select 2 as id, 'goodbye' as msg, 'red' as color + +{% else %} + +-- data for subsequent incremental update + +select 1 as id, 'hey' as msg, 'blue' as color +union all +select 2 as id, 'yo' as msg, 'green' as color +union all +select 3 as id, 'anyway' as msg, 'purple' as color + +{% endif %} +""" incremental_partition_iceberg_sql = """ {{ diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index 18e8a60..6733a3f 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -8,6 +8,9 @@ BaseIncremental, BaseIncrementalNotSchemaChange, ) +from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( + BaseMergeExcludeColumns, +) from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod from dbt.tests.adapter.utils.base_utils import BaseUtils @@ -203,3 +206,7 @@ class TestBaseIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange): @pytest.fixture(scope="class") def models(self): return {"incremental_not_schema_change.sql": incremental_not_schema_change_sql} + + +class TestBaseMergeExcludeColumns(BaseMergeExcludeColumns): + pass diff --git a/tests/functional/adapter/test_iceberg_v1.py b/tests/functional/adapter/test_iceberg_v1.py index 1119c90..08fde52 100644 --- a/tests/functional/adapter/test_iceberg_v1.py +++ b/tests/functional/adapter/test_iceberg_v1.py @@ -23,10 +23,15 @@ ) from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations + from dbt.tests.adapter.basic.test_incremental import ( BaseIncremental, ) +from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( + BaseMergeExcludeColumns, +) + from dbt.tests.adapter.basic.files import ( model_base, model_incremental, @@ -40,6 +45,7 @@ incremental_iceberg_sql, incremental_partition_iceberg_sql, incremental_multiple_partition_iceberg_sql, + merge_iceberg_sql, ) @@ -203,6 +209,12 @@ def models(self): } +class TestMergeIcebergHive(BaseMergeExcludeColumns): + @pytest.fixture(scope="class") + def models(self): + return {"merge_exclude_columns.sql": merge_iceberg_sql} + + class TestIncrementalPartitionIcebergHive(BaseIncrementalForIceberg): @pytest.fixture(scope="class") def models(self): diff --git a/tests/functional/adapter/test_iceberg_v2.py b/tests/functional/adapter/test_iceberg_v2.py index 28858c8..f5cf496 100644 --- a/tests/functional/adapter/test_iceberg_v2.py +++ b/tests/functional/adapter/test_iceberg_v2.py @@ -6,6 +6,7 @@ incremental_iceberg_sql, incremental_partition_iceberg_sql, incremental_multiple_partition_iceberg_sql, + merge_iceberg_sql, ) from tests.functional.adapter.test_iceberg_v1 import ( @@ -13,6 +14,10 @@ BaseIncrementalForIceberg, ) +from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( + BaseMergeExcludeColumns, +) + from dbt.tests.adapter.basic.files import model_base, base_view_sql, schema_base_yml iceberg_base_materialized_var_sql = ( @@ -59,6 +64,12 @@ def models(self): } +class TestMergeIcebergV2Hive(BaseMergeExcludeColumns): + @pytest.fixture(scope="class") + def models(self): + return {"merge_exclude_columns.sql": replace_with_iceberg_v2(merge_iceberg_sql)} + + class TestIncrementalPartitionIcebergV2Hive(BaseIncrementalForIceberg): @pytest.fixture(scope="class") def models(self):