From 491ca76a086198344f71db64a85809929afe0785 Mon Sep 17 00:00:00 2001 From: Vitaly Markov Date: Tue, 17 Dec 2024 21:43:33 +0000 Subject: [PATCH] Initial implementation of ICEBERG_TABLE object type, currently only unmanaged tables are supported --- .github/workflows/pytest.yml | 2 +- CHANGELOG.md | 9 + snowddl/blueprint/__init__.py | 1 + snowddl/blueprint/blueprint.py | 17 +- snowddl/blueprint/object_type.py | 12 +- snowddl/parser/__init__.py | 3 + snowddl/parser/iceberg_table.py | 77 ++++++++ snowddl/parser/schema.py | 14 +- snowddl/resolver/__init__.py | 4 +- snowddl/resolver/iceberg_table.py | 166 ++++++++++++++++++ snowddl/resolver/schema_role.py | 19 ++ snowddl/version.py | 2 +- .../iceberg_sc1/iceberg_table/it001_it1.yaml | 2 + .../step1/iceberg_db1/iceberg_sc1/params.yaml | 4 + .../iceberg_sc2/iceberg_table/it002_it1.yaml | 2 + .../step1/iceberg_db1/iceberg_sc2/params.yaml | 4 + test/_config/step1/permission_model.yaml | 11 ++ .../iceberg_sc1/iceberg_table/it001_it1.yaml | 2 + .../step2/iceberg_db1/iceberg_sc1/params.yaml | 2 + .../iceberg_sc2/iceberg_table/it002_it1.yaml | 2 + .../step2/iceberg_db1/iceberg_sc2/params.yaml | 2 + .../iceberg_sc1/iceberg_table/it001_it2.yaml | 1 + .../step3/iceberg_db1/iceberg_sc1/params.yaml | 2 + .../step3/iceberg_db1/iceberg_sc2/params.yaml | 2 + test/_sql/account_setup.sql | 18 -- test/_sql/iceberg_setup.sql | 49 ++++++ test/conftest.py | 11 ++ test/iceberg_table/it001.py | 33 ++++ test/iceberg_table/it002.py | 33 ++++ test/{run_test.sh => run_test_full.sh} | 0 test/run_test_lite.sh | 36 ++++ 31 files changed, 514 insertions(+), 28 deletions(-) create mode 100644 snowddl/parser/iceberg_table.py create mode 100644 snowddl/resolver/iceberg_table.py create mode 100644 test/_config/step1/iceberg_db1/iceberg_sc1/iceberg_table/it001_it1.yaml create mode 100644 test/_config/step1/iceberg_db1/iceberg_sc1/params.yaml create mode 100644 test/_config/step1/iceberg_db1/iceberg_sc2/iceberg_table/it002_it1.yaml create mode 100644 test/_config/step1/iceberg_db1/iceberg_sc2/params.yaml create mode 100644 test/_config/step2/iceberg_db1/iceberg_sc1/iceberg_table/it001_it1.yaml create mode 100644 test/_config/step2/iceberg_db1/iceberg_sc1/params.yaml create mode 100644 test/_config/step2/iceberg_db1/iceberg_sc2/iceberg_table/it002_it1.yaml create mode 100644 test/_config/step2/iceberg_db1/iceberg_sc2/params.yaml create mode 100644 test/_config/step3/iceberg_db1/iceberg_sc1/iceberg_table/it001_it2.yaml create mode 100644 test/_config/step3/iceberg_db1/iceberg_sc1/params.yaml create mode 100644 test/_config/step3/iceberg_db1/iceberg_sc2/params.yaml create mode 100644 test/_sql/iceberg_setup.sql create mode 100644 test/iceberg_table/it001.py create mode 100644 test/iceberg_table/it002.py rename test/{run_test.sh => run_test_full.sh} (100%) create mode 100755 test/run_test_lite.sh diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index e20934f..d2d8244 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -33,4 +33,4 @@ jobs: run: pip install -e .[dev] - name: Run pytest - run: test/run_test.sh + run: test/run_test_full.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 47bd8d0..b372f02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.38.0] - 2024-12-17 + +- Introduced initial implementation of `ICEBERG_TABLE` object type. Currently only unmanaged Iceberg tables are supported. +- Added parameters `external_volume` and `catalog` for `SCHEMA` object type, required for Iceberg tables to work. +- Split `run_test.sh` script into two scripts: `run_test_full.sh` and `run_test_lite.sh`. The Lite version does not run tests which require complicated setup for external resources. At this moment it skips Iceberg tables. +- Added `iceberg_setup.sql` for tests, helps to prepare environment for Iceberg table tests. + +Managed Iceberg tables will be implemented if we see a sufficient interest from users. + ## [0.37.4] - 2024-12-06 - Relaxed argument validation for `oauth_snowpark` authenticator. diff --git a/snowddl/blueprint/__init__.py b/snowddl/blueprint/__init__.py index d19ca19..7c58dbe 100644 --- a/snowddl/blueprint/__init__.py +++ b/snowddl/blueprint/__init__.py @@ -18,6 +18,7 @@ ForeignKeyBlueprint, FunctionBlueprint, HybridTableBlueprint, + IcebergTableBlueprint, MaterializedViewBlueprint, MaskingPolicyBlueprint, NetworkPolicyBlueprint, diff --git a/snowddl/blueprint/blueprint.py b/snowddl/blueprint/blueprint.py index 0e4e966..f676654 100644 --- a/snowddl/blueprint/blueprint.py +++ b/snowddl/blueprint/blueprint.py @@ -109,9 +109,9 @@ class BusinessRoleBlueprint(AbstractBlueprint): class DatabaseBlueprint(AbstractBlueprint): full_name: DatabaseIdent permission_model: Optional[str] = None + is_sandbox: Optional[bool] = None is_transient: Optional[bool] = None retention_time: Optional[int] = None - is_sandbox: Optional[bool] = None owner_database_write: List[IdentPattern] = [] owner_database_read: List[IdentPattern] = [] owner_integration_usage: List[Ident] = [] @@ -216,6 +216,17 @@ class HybridTableBlueprint(SchemaObjectBlueprint, DependsOnMixin): indexes: Optional[List[IndexReference]] = None +class IcebergTableBlueprint(SchemaObjectBlueprint): + external_volume: Ident + catalog: Ident + catalog_table_name: Optional[str] = None + catalog_namespace: Optional[str] = None + metadata_file_path: Optional[str] = None + base_location: Optional[str] = None + replace_invalid_characters: bool = False + auto_refresh: bool = False + + class MaterializedViewBlueprint(SchemaObjectBlueprint): text: str columns: Optional[List[ViewColumn]] = None @@ -312,9 +323,11 @@ class RowAccessPolicyBlueprint(SchemaObjectBlueprint): class SchemaBlueprint(AbstractBlueprint): full_name: SchemaIdent permission_model: Optional[str] = None + is_sandbox: Optional[bool] = None is_transient: Optional[bool] = None retention_time: Optional[int] = None - is_sandbox: Optional[bool] = None + external_volume: Optional[Ident] = None + catalog: Optional[Ident] = None owner_database_write: List[IdentPattern] = [] owner_database_read: List[IdentPattern] = [] owner_schema_write: List[IdentPattern] = [] diff --git a/snowddl/blueprint/object_type.py b/snowddl/blueprint/object_type.py index 82bc70c..6481775 100644 --- a/snowddl/blueprint/object_type.py +++ b/snowddl/blueprint/object_type.py @@ -98,6 +98,15 @@ class ObjectType(Enum): "blueprint_cls": "ExternalTableBlueprint", } + # Technical object type, used for GRANTs only + # There is no blueprint + EXTERNAL_VOLUME = { + "singular": "EXTERNAL VOLUME", + "plural": "EXTERNAL VOLUMES", + "singular_for_ref": "VOLUME", + "singular_for_grant": "VOLUME", + } + FILE_FORMAT = { "singular": "FILE FORMAT", "plural": "FILE FORMATS", @@ -121,13 +130,12 @@ class ObjectType(Enum): "blueprint_cls": "HybridTableBlueprint", } - # Technical object type, used for GRANTs only - # Currently there is no blueprint ICEBERG_TABLE = { "singular": "ICEBERG TABLE", "plural": "ICEBERG TABLES", "singular_for_ref": "TABLE", "is_future_grant_supported": True, + "blueprint_cls": "IcebergTableBlueprint", } # Technical object type, used for GRANTs only diff --git a/snowddl/parser/__init__.py b/snowddl/parser/__init__.py index 0c11823..c18f5d3 100644 --- a/snowddl/parser/__init__.py +++ b/snowddl/parser/__init__.py @@ -16,6 +16,7 @@ from .file_format import FileFormatParser from .function import FunctionParser from .hybrid_table import HybridTableParser +from .iceberg_table import IcebergTableParser from .materialized_view import MaterializedViewParser from .masking_policy import MaskingPolicyParser from .network_policy import NetworkPolicyParser @@ -68,6 +69,7 @@ TableParser, EventTableParser, HybridTableParser, + IcebergTableParser, DynamicTableParser, ExternalTableParser, StreamParser, @@ -103,6 +105,7 @@ TableParser, EventTableParser, HybridTableParser, + IcebergTableParser, DynamicTableParser, ExternalTableParser, StreamParser, diff --git a/snowddl/parser/iceberg_table.py b/snowddl/parser/iceberg_table.py new file mode 100644 index 0000000..c4be52c --- /dev/null +++ b/snowddl/parser/iceberg_table.py @@ -0,0 +1,77 @@ +from functools import partial + +from snowddl.blueprint import IcebergTableBlueprint, Ident, SchemaObjectIdent +from snowddl.parser.abc_parser import AbstractParser, ParsedFile +from snowddl.parser.schema import schema_json_schema + + +# fmt: off +iceberg_table_json_schema = { + "type": "object", + "properties": { + "catalog_table_name": { + "type": "string" + }, + "catalog_namespace": { + "type": "string", + }, + "metadata_file_path": { + "type": "string" + }, + "base_location": { + "type": "string" + }, + "replace_invalid_characters": { + "type": "boolean" + }, + "auto_refresh": { + "type": "boolean" + }, + "comment": { + "type": "string" + }, + }, + "oneOf": [ + {"required": ["catalog_table_name"]}, + {"required": ["metadata_file_path"]}, + {"required": ["base_location"]}, + ], + "additionalProperties": False +} +# fmt: on + + +class IcebergTableParser(AbstractParser): + def load_blueprints(self): + combined_params = {} + + for database_name in self.get_database_names(): + combined_params[database_name] = {} + + for schema_name in self.get_schema_names_in_database(database_name): + schema_params = self.parse_single_entity_file(f"{database_name}/{schema_name}/params", schema_json_schema) + combined_params[database_name][schema_name] = schema_params + + self.parse_schema_object_files("iceberg_table", iceberg_table_json_schema, partial(self.process_table, combined_params=combined_params)) + + def process_table(self, f: ParsedFile, combined_params: dict): + if not combined_params[f.database][f.schema].get("external_volume"): + raise ValueError("Iceberg table requires parameter [external_volume] to be defined on schema level") + + if not combined_params[f.database][f.schema].get("catalog"): + raise ValueError("Iceberg table requires parameter [catalog] to be defined on schema level") + + bp = IcebergTableBlueprint( + full_name=SchemaObjectIdent(self.env_prefix, f.database, f.schema, f.name), + external_volume=Ident(combined_params[f.database][f.schema].get("external_volume")), + catalog=Ident(combined_params[f.database][f.schema].get("catalog")), + catalog_table_name=f.params.get("catalog_table_name"), + catalog_namespace=f.params.get("catalog_namespace"), + metadata_file_path=f.params.get("metadata_file_path"), + base_location=f.params.get("base_location"), + replace_invalid_characters=f.params.get("replace_invalid_characters", False), + auto_refresh=f.params.get("auto_refresh", False), + comment=f.params.get("comment"), + ) + + self.config.add_blueprint(bp) diff --git a/snowddl/parser/schema.py b/snowddl/parser/schema.py index 3faf0c8..3543726 100644 --- a/snowddl/parser/schema.py +++ b/snowddl/parser/schema.py @@ -18,14 +18,20 @@ "permission_model": { "type": "string", }, + "is_sandbox": { + "type": "boolean" + }, "is_transient": { "type": "boolean" }, "retention_time": { "type": "integer" }, - "is_sandbox": { - "type": "boolean" + "external_volume": { + "type": "string" + }, + "catalog": { + "type": "string", }, "owner_database_read": { "type": "array", @@ -112,9 +118,11 @@ def load_blueprints(self): bp = SchemaBlueprint( full_name=SchemaIdent(self.env_prefix, database_name, schema_name), permission_model=schema_permission_model_name, + is_sandbox=combined_params.get("is_sandbox", False), is_transient=combined_params.get("is_transient", False), retention_time=combined_params.get("retention_time", None), - is_sandbox=combined_params.get("is_sandbox", False), + external_volume=Ident(schema_params.get("external_volume")) if schema_params.get("external_volume") else None, + catalog=Ident(schema_params.get("catalog")) if schema_params.get("catalog") else None, owner_database_write=[IdentPattern(p) for p in schema_params.get("owner_database_write", [])], owner_database_read=[IdentPattern(p) for p in schema_params.get("owner_database_read", [])], owner_schema_write=[IdentPattern(p) for p in schema_params.get("owner_schema_write", [])], diff --git a/snowddl/resolver/__init__.py b/snowddl/resolver/__init__.py index eee22cf..35f8f69 100644 --- a/snowddl/resolver/__init__.py +++ b/snowddl/resolver/__init__.py @@ -18,6 +18,7 @@ from .foreign_key import ForeignKeyResolver from .function import FunctionResolver from .hybrid_table import HybridTableResolver +from .iceberg_table import IcebergTableResolver from .masking_policy import MaskingPolicyResolver from .materialized_view import MaterializedViewResolver from .network_policy import NetworkPolicyResolver @@ -72,6 +73,7 @@ TableResolver, EventTableResolver, HybridTableResolver, + IcebergTableResolver, DynamicTableResolver, ExternalTableResolver, PrimaryKeyResolver, @@ -138,8 +140,8 @@ TableResolver, EventTableResolver, HybridTableResolver, + IcebergTableResolver, DynamicTableResolver, - ExternalTableResolver, PrimaryKeyResolver, UniqueKeyResolver, ForeignKeyResolver, diff --git a/snowddl/resolver/iceberg_table.py b/snowddl/resolver/iceberg_table.py new file mode 100644 index 0000000..45bd774 --- /dev/null +++ b/snowddl/resolver/iceberg_table.py @@ -0,0 +1,166 @@ +from snowddl.blueprint import IcebergTableBlueprint +from snowddl.resolver.abc_schema_object_resolver import AbstractSchemaObjectResolver, ResolveResult, ObjectType + + +class IcebergTableResolver(AbstractSchemaObjectResolver): + skip_on_empty_blueprints = True + + def get_object_type(self) -> ObjectType: + return ObjectType.ICEBERG_TABLE + + def get_existing_objects_in_schema(self, schema: dict): + existing_objects = {} + + cur = self.engine.execute_meta( + "SHOW ICEBERG TABLES IN SCHEMA {database:i}.{schema:i}", + { + "database": schema["database"], + "schema": schema["schema"], + }, + ) + + for r in cur: + # Currently only external iceberg tables are supported + if r["iceberg_table_type"] != "UNMANAGED": + continue + + existing_objects[f"{r['database_name']}.{r['schema_name']}.{r['name']}"] = { + "database": r["database_name"], + "schema": r["schema_name"], + "name": r["name"], + "owner": r["owner"], + "comment": r["comment"] if r["comment"] else None, + } + + return existing_objects + + def get_blueprints(self): + return self.config.get_blueprints_by_type(IcebergTableBlueprint) + + def create_object(self, bp: IcebergTableBlueprint): + create_query = self.engine.query_builder() + common_query = self._build_common_unmanaged_iceberg_table_sql(bp) + + create_query.append( + "CREATE ICEBERG TABLE {full_name:i}", + { + "full_name": bp.full_name, + } + ) + + create_query.append_nl(common_query) + + self.engine.execute_safe_ddl(create_query) + self.engine.execute_safe_ddl( + "ALTER ICEBERG TABLE {full_name:i} SET COMMENT = {comment}", + { + "full_name": bp.full_name, + "comment": common_query.add_short_hash(bp.comment), + }, + ) + + return ResolveResult.CREATE + + def compare_object(self, bp: IcebergTableBlueprint, row: dict): + replace_query = self.engine.query_builder() + common_query = self._build_common_unmanaged_iceberg_table_sql(bp) + + if not common_query.compare_short_hash(row["comment"]): + replace_query.append( + "CREATE OR REPLACE ICEBERG TABLE {full_name:i}", + { + "full_name": bp.full_name, + } + ) + + replace_query.append_nl(common_query) + + self.engine.execute_safe_ddl(replace_query) + self.engine.execute_safe_ddl( + "ALTER ICEBERG TABLE {full_name:i} SET COMMENT = {comment}", + { + "full_name": bp.full_name, + "comment": common_query.add_short_hash(bp.comment), + }, + ) + + return ResolveResult.REPLACE + + return ResolveResult.NOCHANGE + + def drop_object(self, row: dict): + self.engine.execute_unsafe_ddl( + "DROP ICEBERG TABLE {database:i}.{schema:i}.{name:i}", + { + "database": row["database"], + "schema": row["schema"], + "name": row["name"], + }, + ) + + return ResolveResult.DROP + + def _build_common_unmanaged_iceberg_table_sql(self, bp: IcebergTableBlueprint): + query = self.engine.query_builder() + + query.append_nl( + "EXTERNAL_VOLUME = {external_volume:i}", + { + "external_volume": bp.external_volume, + } + ) + + query.append_nl( + "CATALOG = {catalog:i}", + { + "catalog": bp.catalog, + } + ) + + if bp.catalog_table_name: + query.append_nl( + "CATALOG_TABLE_NAME = {catalog_table_name}", + { + "catalog_table_name": bp.catalog_table_name, + } + ) + + if bp.catalog_namespace: + query.append_nl( + "CATALOG_NAMESPACE = {catalog_namespace}", + { + "catalog_namespace": bp.catalog_namespace, + } + ) + + if bp.metadata_file_path: + query.append_nl( + "METADATA_FILE_PATH = {metadata_file_path}", + { + "metadata_file_path": bp.metadata_file_path, + } + ) + + if bp.base_location: + query.append_nl( + "BASE_LOCATION = {base_location}", + { + "base_location": bp.base_location, + } + ) + + if bp.replace_invalid_characters: + query.append_nl("REPLACE_INVALID_CHARACTERS = TRUE") + + if bp.auto_refresh: + query.append_nl("AUTO_REFRESH = TRUE") + + if bp.comment: + query.append_nl( + "COMMENT = {comment}", + { + "comment": bp.comment + } + ) + + return query diff --git a/snowddl/resolver/schema_role.py b/snowddl/resolver/schema_role.py index 1504262..8d85e40 100644 --- a/snowddl/resolver/schema_role.py +++ b/snowddl/resolver/schema_role.py @@ -55,6 +55,25 @@ def get_blueprint_owner_role(self, schema_bp: SchemaBlueprint): ) ) + # Iceberg-related grants + if schema_bp.external_volume: + grants.append( + Grant( + privilege="USAGE", + on=ObjectType.EXTERNAL_VOLUME, + name=schema_bp.external_volume, + ) + ) + + if schema_bp.catalog: + grants.append( + Grant( + privilege="USAGE", + on=ObjectType.INTEGRATION, + name=schema_bp.catalog, + ) + ) + # Create grants for model_create_grant in schema_permission_model.owner_create_grants: grants.append( diff --git a/snowddl/version.py b/snowddl/version.py index 2b90be6..978180b 100644 --- a/snowddl/version.py +++ b/snowddl/version.py @@ -1 +1 @@ -__version__ = "0.37.4" +__version__ = "0.38.0" diff --git a/test/_config/step1/iceberg_db1/iceberg_sc1/iceberg_table/it001_it1.yaml b/test/_config/step1/iceberg_db1/iceberg_sc1/iceberg_table/it001_it1.yaml new file mode 100644 index 0000000..f957ba5 --- /dev/null +++ b/test/_config/step1/iceberg_db1/iceberg_sc1/iceberg_table/it001_it1.yaml @@ -0,0 +1,2 @@ +catalog_table_name: test_iceberg_table_1 +comment: abc diff --git a/test/_config/step1/iceberg_db1/iceberg_sc1/params.yaml b/test/_config/step1/iceberg_db1/iceberg_sc1/params.yaml new file mode 100644 index 0000000..c69dfe2 --- /dev/null +++ b/test/_config/step1/iceberg_db1/iceberg_sc1/params.yaml @@ -0,0 +1,4 @@ +permission_model: iceberg + +external_volume: test_external_volume_glue +catalog: test_catalog_glue diff --git a/test/_config/step1/iceberg_db1/iceberg_sc2/iceberg_table/it002_it1.yaml b/test/_config/step1/iceberg_db1/iceberg_sc2/iceberg_table/it002_it1.yaml new file mode 100644 index 0000000..cbfc287 --- /dev/null +++ b/test/_config/step1/iceberg_db1/iceberg_sc2/iceberg_table/it002_it1.yaml @@ -0,0 +1,2 @@ +metadata_file_path: test_iceberg_table_1/metadata/00001-cc112050-1448-4c2a-9e03-504e7f5fc62a.metadata.json +replace_invalid_characters: true diff --git a/test/_config/step1/iceberg_db1/iceberg_sc2/params.yaml b/test/_config/step1/iceberg_db1/iceberg_sc2/params.yaml new file mode 100644 index 0000000..fbfb918 --- /dev/null +++ b/test/_config/step1/iceberg_db1/iceberg_sc2/params.yaml @@ -0,0 +1,4 @@ +permission_model: iceberg + +external_volume: test_external_volume_glue +catalog: test_catalog_object_store diff --git a/test/_config/step1/permission_model.yaml b/test/_config/step1/permission_model.yaml index ebe6be5..f6d5d6a 100644 --- a/test/_config/step1/permission_model.yaml +++ b/test/_config/step1/permission_model.yaml @@ -5,3 +5,14 @@ database_owner_model: schema_owner_model: inherit_from: default ruleset: SCHEMA_OWNER + +iceberg: + inherit_from: default + owner_create_grants: + - ICEBERG_TABLE + owner_future_grants: + ICEBERG_TABLE: [OWNERSHIP] + write_future_grants: + ICEBERG_TABLE: [INSERT, UPDATE, DELETE, TRUNCATE] + read_future_grants: + ICEBERG_TABLE: [SELECT, REFERENCES] diff --git a/test/_config/step2/iceberg_db1/iceberg_sc1/iceberg_table/it001_it1.yaml b/test/_config/step2/iceberg_db1/iceberg_sc1/iceberg_table/it001_it1.yaml new file mode 100644 index 0000000..88390c6 --- /dev/null +++ b/test/_config/step2/iceberg_db1/iceberg_sc1/iceberg_table/it001_it1.yaml @@ -0,0 +1,2 @@ +catalog_table_name: test_iceberg_table_2 +comment: cde diff --git a/test/_config/step2/iceberg_db1/iceberg_sc1/params.yaml b/test/_config/step2/iceberg_db1/iceberg_sc1/params.yaml new file mode 100644 index 0000000..098d88d --- /dev/null +++ b/test/_config/step2/iceberg_db1/iceberg_sc1/params.yaml @@ -0,0 +1,2 @@ +external_volume: test_external_volume_glue +catalog: test_catalog_glue diff --git a/test/_config/step2/iceberg_db1/iceberg_sc2/iceberg_table/it002_it1.yaml b/test/_config/step2/iceberg_db1/iceberg_sc2/iceberg_table/it002_it1.yaml new file mode 100644 index 0000000..d3525c5 --- /dev/null +++ b/test/_config/step2/iceberg_db1/iceberg_sc2/iceberg_table/it002_it1.yaml @@ -0,0 +1,2 @@ +metadata_file_path: test_iceberg_table_2/metadata/00000-c8b82a2f-9504-4bed-8050-af8ec14afa26.metadata.json +replace_invalid_characters: true diff --git a/test/_config/step2/iceberg_db1/iceberg_sc2/params.yaml b/test/_config/step2/iceberg_db1/iceberg_sc2/params.yaml new file mode 100644 index 0000000..c9137ec --- /dev/null +++ b/test/_config/step2/iceberg_db1/iceberg_sc2/params.yaml @@ -0,0 +1,2 @@ +external_volume: test_external_volume_glue +catalog: test_catalog_object_store diff --git a/test/_config/step3/iceberg_db1/iceberg_sc1/iceberg_table/it001_it2.yaml b/test/_config/step3/iceberg_db1/iceberg_sc1/iceberg_table/it001_it2.yaml new file mode 100644 index 0000000..b3ab368 --- /dev/null +++ b/test/_config/step3/iceberg_db1/iceberg_sc1/iceberg_table/it001_it2.yaml @@ -0,0 +1 @@ +catalog_table_name: test_iceberg_table_2 diff --git a/test/_config/step3/iceberg_db1/iceberg_sc1/params.yaml b/test/_config/step3/iceberg_db1/iceberg_sc1/params.yaml new file mode 100644 index 0000000..098d88d --- /dev/null +++ b/test/_config/step3/iceberg_db1/iceberg_sc1/params.yaml @@ -0,0 +1,2 @@ +external_volume: test_external_volume_glue +catalog: test_catalog_glue diff --git a/test/_config/step3/iceberg_db1/iceberg_sc2/params.yaml b/test/_config/step3/iceberg_db1/iceberg_sc2/params.yaml new file mode 100644 index 0000000..c9137ec --- /dev/null +++ b/test/_config/step3/iceberg_db1/iceberg_sc2/params.yaml @@ -0,0 +1,2 @@ +external_volume: test_external_volume_glue +catalog: test_catalog_object_store diff --git a/test/_sql/account_setup.sql b/test/_sql/account_setup.sql index 87f5866..be66d53 100644 --- a/test/_sql/account_setup.sql +++ b/test/_sql/account_setup.sql @@ -3,12 +3,6 @@ SET PASSWORD = ''; --- Follow instructions to set up AWS role ARN: https://docs.snowflake.com/en/user-guide/data-load-s3-config-storage-integration --- You may use UUID for EXTERNAL_ID - -SET STORAGE_AWS_ROLE_ARN = ''; -- replace with AWS role ARN for STORAGE INTEGRATION -SET STORAGE_AWS_EXTERNAL_ID = ''; -- replace with randomly generated EXTERNAL_ID - --- USE ROLE ACCOUNTADMIN; @@ -59,18 +53,6 @@ GRANT OVERRIDE SHARE RESTRICTIONS ON ACCOUNT TO ROLE SNOWDDL_ADMIN; --- -CREATE STORAGE INTEGRATION TEST_STORAGE_INTEGRATION_AWS -TYPE = EXTERNAL_STAGE -STORAGE_PROVIDER = 'S3' -ENABLED = TRUE -STORAGE_AWS_ROLE_ARN = $STORAGE_AWS_ROLE_ARN -STORAGE_ALLOWED_LOCATIONS = ('*') -; - -GRANT USAGE ON INTEGRATION TEST_STORAGE_INTEGRATION_AWS TO ROLE SNOWDDL_ADMIN; - ---- - CREATE STORAGE INTEGRATION TEST_STORAGE_INTEGRATION_GCP TYPE = EXTERNAL_STAGE STORAGE_PROVIDER = 'GCS' diff --git a/test/_sql/iceberg_setup.sql b/test/_sql/iceberg_setup.sql new file mode 100644 index 0000000..ad571d4 --- /dev/null +++ b/test/_sql/iceberg_setup.sql @@ -0,0 +1,49 @@ +-- Configure external volume: +-- https://docs.snowflake.com/en/user-guide/tables-iceberg-configure-external-volume-s3 +SET STORAGE_BASE_URL = 's3://snowddl-test-1/iceberg_glue/'; +SET STORAGE_ROLE_ARN = 'arn:aws:iam::571600836355:role/snowflake_role'; +SET STORAGE_EXTERNAL_ID = 'BKB93946_SFCRole=589_NdIqHV/NfOf1EcQ1SuEWXxP6MoA='; + +-- Configure catalog integration: +-- https://docs.snowflake.com/en/user-guide/tables-iceberg-configure-catalog-integration-glue +SET GLUE_CATALOG_NAMESPACE = 'iceberg_glue' +SET GLUE_ROLE_ARN = 'arn:aws:iam::571600836355:role/snowflake_glue'; +SET GLUE_CATALOG_ID = '571600836355'; +SET GLUE_REGION = 'us-east-1'; + +--- + +USE ROLE ACCOUNTADMIN; + +--- + +CREATE OR REPLACE EXTERNAL VOLUME TEST_EXTERNAL_VOLUME_GLUE +STORAGE_LOCATIONS = +( + ( + NAME = 'iceberg_glue' + STORAGE_PROVIDER = 'S3' + STORAGE_BASE_URL = $STORAGE_BASE_URL + STORAGE_AWS_ROLE_ARN = $STORAGE_ROLE_ARN + STORAGE_AWS_EXTERNAL_ID = $STORAGE_EXTERNAL_ID + ) +) +ALLOW_WRITES = FALSE; + +--- + +CREATE OR REPLACE CATALOG INTEGRATION TEST_CATALOG_GLUE +CATALOG_SOURCE = GLUE +CATALOG_NAMESPACE = $GLUE_CATALOG_NAMESPACE +TABLE_FORMAT = ICEBERG +GLUE_AWS_ROLE_ARN = $GLUE_ROLE_ARN +GLUE_CATALOG_ID = $GLUE_CATALOG_ID +GLUE_REGION = $GLUE_REGION +ENABLED = TRUE; + +CREATE OR REPLACE CATALOG INTEGRATION TEST_CATALOG_OBJECT_STORE +CATALOG_SOURCE = OBJECT_STORE +TABLE_FORMAT = ICEBERG +ENABLED = TRUE; + +--- diff --git a/test/conftest.py b/test/conftest.py index 1cebfa9..8362f30 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -439,6 +439,17 @@ def show_hybrid_table(self, database, schema, name): return cur.fetchone() + def show_iceberg_table(self, database, schema, name): + cur = self.execute( + "SHOW ICEBERG TABLES LIKE {object_name:lf} IN SCHEMA {schema_name:i}", + { + "schema_name": SchemaIdent(self.env_prefix, database, schema), + "object_name": Ident(name), + }, + ) + + return cur.fetchone() + def show_indexes(self, database, schema, name): cur = self.execute( "SHOW INDEXES IN TABLE {table_name:i}", diff --git a/test/iceberg_table/it001.py b/test/iceberg_table/it001.py new file mode 100644 index 0000000..0712796 --- /dev/null +++ b/test/iceberg_table/it001.py @@ -0,0 +1,33 @@ +import os +import pytest + +pytestmark = pytest.mark.skipif(os.environ.get("TEST_LITE"), reason="Tests require additional setup for Iceberg tables") + + +def test_step1(helper): + iceberg_table_show = helper.show_iceberg_table("iceberg_db1", "iceberg_sc1", "it001_it1") + + assert iceberg_table_show["iceberg_table_type"] == "UNMANAGED" + assert iceberg_table_show["external_volume_name"] == "TEST_EXTERNAL_VOLUME_GLUE" + assert iceberg_table_show["catalog_name"] == "TEST_CATALOG_GLUE" + + assert iceberg_table_show["catalog_table_name"] == "test_iceberg_table_1" + assert iceberg_table_show["comment"].startswith("abc #") + + +def test_step2(helper): + iceberg_table_show = helper.show_iceberg_table("iceberg_db1", "iceberg_sc1", "it001_it1") + + assert iceberg_table_show["iceberg_table_type"] == "UNMANAGED" + assert iceberg_table_show["external_volume_name"] == "TEST_EXTERNAL_VOLUME_GLUE" + assert iceberg_table_show["catalog_name"] == "TEST_CATALOG_GLUE" + + assert iceberg_table_show["catalog_table_name"] == "test_iceberg_table_2" + assert iceberg_table_show["comment"].startswith("cde #") + + +def test_step3(helper): + iceberg_table_show = helper.show_iceberg_table("iceberg_db1", "iceberg_sc1", "it001_it1") + + # Table was dropped + assert iceberg_table_show is None diff --git a/test/iceberg_table/it002.py b/test/iceberg_table/it002.py new file mode 100644 index 0000000..b530baf --- /dev/null +++ b/test/iceberg_table/it002.py @@ -0,0 +1,33 @@ +import os +import pytest + +pytestmark = pytest.mark.skipif(os.environ.get("TEST_LITE"), reason="Tests require additional setup for Iceberg tables") + + +def test_step1(helper): + iceberg_table_show = helper.show_iceberg_table("iceberg_db1", "iceberg_sc2", "it002_it1") + + assert iceberg_table_show["iceberg_table_type"] == "UNMANAGED" + assert iceberg_table_show["external_volume_name"] == "TEST_EXTERNAL_VOLUME_GLUE" + assert iceberg_table_show["catalog_name"] == "TEST_CATALOG_OBJECT_STORE" + + assert iceberg_table_show["catalog_table_name"] is None + assert iceberg_table_show["comment"].startswith("#") + + +def test_step2(helper): + iceberg_table_show = helper.show_iceberg_table("iceberg_db1", "iceberg_sc2", "it002_it1") + + assert iceberg_table_show["iceberg_table_type"] == "UNMANAGED" + assert iceberg_table_show["external_volume_name"] == "TEST_EXTERNAL_VOLUME_GLUE" + assert iceberg_table_show["catalog_name"] == "TEST_CATALOG_OBJECT_STORE" + + assert iceberg_table_show["catalog_table_name"] is None + assert iceberg_table_show["comment"].startswith("#") + + +def test_step3(helper): + iceberg_table_show = helper.show_iceberg_table("iceberg_db1", "iceberg_sc2", "it002_it1") + + # Table was dropped + assert iceberg_table_show is None diff --git a/test/run_test.sh b/test/run_test_full.sh similarity index 100% rename from test/run_test.sh rename to test/run_test_full.sh diff --git a/test/run_test_lite.sh b/test/run_test_lite.sh new file mode 100755 index 0000000..2fad9ba --- /dev/null +++ b/test/run_test_lite.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# Do not resolve objects which require setup of external resources +EXCLUDE_OBJECT_TYPES="ICEBERG_TABLE" + +# Set the following environment variables: +# - SNOWFLAKE_ACCOUNT +# - SNOWFLAKE_USER +# - SNOWFLAKE_PASSWORD +# - SNOWFLAKE_ENV_PREFIX +# - SNOWFLAKE_ENV_ADMIN_ROLE + +cd "${0%/*}" + +# Cleanup before +snowddl -c _config/step1 --exclude-object-types=$EXCLUDE_OBJECT_TYPES --apply-unsafe --apply-resource-monitor --apply-all-policy destroy + +# Apply step1 +snowddl -c _config/step1 --exclude-object-types=$EXCLUDE_OBJECT_TYPES --apply-unsafe --apply-resource-monitor --apply-all-policy apply + +# Run test step1 +TEST_LITE=1 pytest -k "step1" --tb=short */*.py + +# Apply step2 +snowddl -c _config/step2 --exclude-object-types=$EXCLUDE_OBJECT_TYPES --apply-unsafe --apply-replace-table --apply-resource-monitor --apply-all-policy --refresh-stage-encryption --refresh-secrets apply + +# Run test step2 +TEST_LITE=1 pytest -k "step2" --tb=short */*.py + +# Apply step3 +snowddl -c _config/step3 --exclude-object-types=$EXCLUDE_OBJECT_TYPES --apply-unsafe --apply-replace-table --apply-resource-monitor --apply-all-policy --refresh-stage-encryption --refresh-secrets apply + +# Run test step3 +TEST_LITE=1 pytest -k "step3" --tb=short */*.py + +# Cleanup after +snowddl -c _config/step1 --exclude-object-types=$EXCLUDE_OBJECT_TYPES --apply-unsafe --apply-resource-monitor --apply-all-policy destroy