From 549fcc875f2111bd2b138262ca087e6c75cb0a99 Mon Sep 17 00:00:00 2001 From: Vitaly Markov Date: Wed, 12 Jun 2024 15:36:45 +0100 Subject: [PATCH] Implement AGGREGATION_POLICY and PROJECTION_POLICY; Add exempt_other_policies for MASKING_POLICY; Add test objects for policies; Reformat some code --- CHANGELOG.md | 7 + snowddl/app/base.py | 34 ++- snowddl/app/singledb.py | 15 ++ snowddl/blueprint/__init__.py | 12 +- snowddl/blueprint/blueprint.py | 21 +- snowddl/blueprint/object_type.py | 12 + snowddl/blueprint/reference.py | 12 + snowddl/config.py | 6 +- snowddl/parser/__init__.py | 6 + snowddl/parser/aggregation_policy.py | 76 ++++++ snowddl/parser/database.py | 2 + snowddl/parser/masking_policy.py | 4 + snowddl/parser/projection_policy.py | 72 ++++++ snowddl/parser/schema.py | 2 + snowddl/resolver/__init__.py | 6 + snowddl/resolver/aggregation_policy.py | 227 ++++++++++++++++++ snowddl/resolver/masking_policy.py | 14 +- snowddl/resolver/projection_policy.py | 212 ++++++++++++++++ snowddl/settings.py | 2 + snowddl/version.py | 2 +- .../db1/sc1/aggregation_policy/ap001_ap1.yaml | 13 + .../db1/sc1/masking_policy/mp001_mp1.yaml | 21 ++ .../db1/sc1/projection_policy/pp001_pp1.yaml | 19 ++ .../db1/sc1/row_access_policy/rp001_rp1.yaml | 16 ++ .../step1/db1/sc1/table/ap001_tb1.yaml | 3 + .../step1/db1/sc1/table/mp001_tb1.yaml | 3 + .../step1/db1/sc1/table/pp001_tb1.yaml | 3 + .../step1/db1/sc1/table/rp001_tb1.yaml | 3 + .../_config/step1/db1/sc1/view/ap001_vw1.yaml | 3 + .../_config/step1/db1/sc1/view/mp001_vw1.yaml | 2 + .../_config/step1/db1/sc1/view/pp001_vw1.yaml | 2 + .../_config/step1/db1/sc1/view/rp001_vw1.yaml | 3 + .../db1/sc1/aggregation_policy/ap001_ap1.yaml | 15 ++ .../db1/sc1/masking_policy/mp001_mp1.yaml | 23 ++ .../db1/sc1/projection_policy/pp001_pp1.yaml | 15 ++ .../db1/sc1/row_access_policy/rp001_rp1.yaml | 12 + .../step2/db1/sc1/table/ap001_tb1.yaml | 3 + .../step2/db1/sc1/table/mp001_tb1.yaml | 3 + .../step2/db1/sc1/table/pp001_tb1.yaml | 3 + .../step2/db1/sc1/table/rp001_tb1.yaml | 3 + .../_config/step2/db1/sc1/view/ap001_vw1.yaml | 3 + .../_config/step2/db1/sc1/view/mp001_vw1.yaml | 2 + .../_config/step2/db1/sc1/view/pp001_vw1.yaml | 2 + .../_config/step2/db1/sc1/view/rp001_vw1.yaml | 3 + .../db1/sc1/aggregation_policy/ap001_ap2.yaml | 2 + .../db1/sc1/masking_policy/mp001_mp2.yaml | 7 + .../db1/sc1/projection_policy/pp001_pp2.yaml | 2 + .../db1/sc1/row_access_policy/rp001_rp2.yaml | 5 + .../step3/db1/sc1/table/ap001_tb1.yaml | 3 + .../step3/db1/sc1/table/mp001_tb1.yaml | 3 + .../step3/db1/sc1/table/pp001_tb1.yaml | 3 + .../step3/db1/sc1/table/rp001_tb1.yaml | 3 + .../_config/step3/db1/sc1/view/ap001_vw1.yaml | 3 + .../_config/step3/db1/sc1/view/mp001_vw1.yaml | 2 + .../_config/step3/db1/sc1/view/pp001_vw1.yaml | 2 + .../_config/step3/db1/sc1/view/rp001_vw1.yaml | 3 + test/run_test.sh | 8 +- 57 files changed, 955 insertions(+), 13 deletions(-) create mode 100644 snowddl/parser/aggregation_policy.py create mode 100644 snowddl/parser/projection_policy.py create mode 100644 snowddl/resolver/aggregation_policy.py create mode 100644 snowddl/resolver/projection_policy.py create mode 100644 test/_config/step1/db1/sc1/aggregation_policy/ap001_ap1.yaml create mode 100644 test/_config/step1/db1/sc1/masking_policy/mp001_mp1.yaml create mode 100644 test/_config/step1/db1/sc1/projection_policy/pp001_pp1.yaml create mode 100644 test/_config/step1/db1/sc1/row_access_policy/rp001_rp1.yaml create mode 100644 test/_config/step1/db1/sc1/table/ap001_tb1.yaml create mode 100644 test/_config/step1/db1/sc1/table/mp001_tb1.yaml create mode 100644 test/_config/step1/db1/sc1/table/pp001_tb1.yaml create mode 100644 test/_config/step1/db1/sc1/table/rp001_tb1.yaml create mode 100644 test/_config/step1/db1/sc1/view/ap001_vw1.yaml create mode 100644 test/_config/step1/db1/sc1/view/mp001_vw1.yaml create mode 100644 test/_config/step1/db1/sc1/view/pp001_vw1.yaml create mode 100644 test/_config/step1/db1/sc1/view/rp001_vw1.yaml create mode 100644 test/_config/step2/db1/sc1/aggregation_policy/ap001_ap1.yaml create mode 100644 test/_config/step2/db1/sc1/masking_policy/mp001_mp1.yaml create mode 100644 test/_config/step2/db1/sc1/projection_policy/pp001_pp1.yaml create mode 100644 test/_config/step2/db1/sc1/row_access_policy/rp001_rp1.yaml create mode 100644 test/_config/step2/db1/sc1/table/ap001_tb1.yaml create mode 100644 test/_config/step2/db1/sc1/table/mp001_tb1.yaml create mode 100644 test/_config/step2/db1/sc1/table/pp001_tb1.yaml create mode 100644 test/_config/step2/db1/sc1/table/rp001_tb1.yaml create mode 100644 test/_config/step2/db1/sc1/view/ap001_vw1.yaml create mode 100644 test/_config/step2/db1/sc1/view/mp001_vw1.yaml create mode 100644 test/_config/step2/db1/sc1/view/pp001_vw1.yaml create mode 100644 test/_config/step2/db1/sc1/view/rp001_vw1.yaml create mode 100644 test/_config/step3/db1/sc1/aggregation_policy/ap001_ap2.yaml create mode 100644 test/_config/step3/db1/sc1/masking_policy/mp001_mp2.yaml create mode 100644 test/_config/step3/db1/sc1/projection_policy/pp001_pp2.yaml create mode 100644 test/_config/step3/db1/sc1/row_access_policy/rp001_rp2.yaml create mode 100644 test/_config/step3/db1/sc1/table/ap001_tb1.yaml create mode 100644 test/_config/step3/db1/sc1/table/mp001_tb1.yaml create mode 100644 test/_config/step3/db1/sc1/table/pp001_tb1.yaml create mode 100644 test/_config/step3/db1/sc1/table/rp001_tb1.yaml create mode 100644 test/_config/step3/db1/sc1/view/ap001_vw1.yaml create mode 100644 test/_config/step3/db1/sc1/view/mp001_vw1.yaml create mode 100644 test/_config/step3/db1/sc1/view/pp001_vw1.yaml create mode 100644 test/_config/step3/db1/sc1/view/rp001_vw1.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 861d6d5..8e309e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.29.0] - 2024-06-12 + +- Implemented `AGGREGATION_POLICY`, `PROJECTION_POLICY` object types. +- Added property `exempt_other_policies` for `MASKING_POLICY`. +- Added CLI option `--apply-all-policy` to execute SQL for all types of policies. +- Prepared test objects for all types of policies. + ## [0.28.3] - 2024-06-08 - Implemented graceful warning when encounter identifier which does not conform to SnowDDL standards while processing existing role grants. Previously it caused SnowDDL to stop with hard error. diff --git a/snowddl/app/base.py b/snowddl/app/base.py index 1bc7387..8764c7d 100644 --- a/snowddl/app/base.py +++ b/snowddl/app/base.py @@ -157,9 +157,24 @@ def init_arguments_parser(self): default=False, action="store_true", ) + parser.add_argument( + "--apply-all-policy", help="Additionally apply changes to all types of POLICIES", default=False, action="store_true" + ) + parser.add_argument( + "--apply-aggregation-policy", + help="Additionally apply changes to AGGREGATION POLICIES", + default=False, + action="store_true", + ) parser.add_argument( "--apply-masking-policy", help="Additionally apply changes to MASKING POLICIES", default=False, action="store_true" ) + parser.add_argument( + "--apply-projection-policy", + help="Additionally apply changes to PROJECTION POLICIES", + default=False, + action="store_true", + ) parser.add_argument( "--apply-row-access-policy", help="Additionally apply changes to ROW ACCESS POLICIES", @@ -337,9 +352,22 @@ def init_settings(self): if self.args.get("apply_replace_table"): settings.execute_replace_table = True + if self.args.get("apply_all_policy"): + settings.execute_aggregation_policy = True + settings.execute_masking_policy = True + settings.execute_projection_policy = True + settings.execute_row_access_policy = True + settings.execute_network_policy = True + + if self.args.get("apply_aggregation_policy"): + settings.execute_aggregation_policy = True + if self.args.get("apply_masking_policy"): settings.execute_masking_policy = True + if self.args.get("apply_projection_policy"): + settings.execute_projection_policy = True + if self.args.get("apply_row_access_policy"): settings.execute_row_access_policy = True @@ -543,8 +571,10 @@ def output_engine_stats(self): def output_engine_warnings(self): for object_type, object_names in self.engine.intention_cache.invalid_name_warning.items(): for name in object_names: - self.logger.warning(f"Detected {object_type.name} with name [{name}] " - f"which does not conform to SnowDDL standards, please rename or drop it manually") + self.logger.warning( + f"Detected {object_type.name} with name [{name}] " + f"which does not conform to SnowDDL standards, please rename or drop it manually" + ) def output_suggested_ddl(self): if self.engine.suggested_ddl: diff --git a/snowddl/app/singledb.py b/snowddl/app/singledb.py index f970aad..b0fd17d 100644 --- a/snowddl/app/singledb.py +++ b/snowddl/app/singledb.py @@ -152,9 +152,24 @@ def init_arguments_parser(self): default=False, action="store_true", ) + parser.add_argument( + "--apply-all-policy", help="Additionally apply changes to all types of POLICIES", default=False, action="store_true" + ) + parser.add_argument( + "--apply-aggregation-policy", + help="Additionally apply changes to AGGREGATION POLICIES", + default=False, + action="store_true", + ) parser.add_argument( "--apply-masking-policy", help="Additionally apply changes to MASKING POLICIES", default=False, action="store_true" ) + parser.add_argument( + "--apply-projection-policy", + help="Additionally apply changes to PROJECTION POLICIES", + default=False, + action="store_true", + ) parser.add_argument( "--apply-row-access-policy", help="Additionally apply changes to ROW ACCESS POLICIES", diff --git a/snowddl/blueprint/__init__.py b/snowddl/blueprint/__init__.py index ffe0356..08900f0 100644 --- a/snowddl/blueprint/__init__.py +++ b/snowddl/blueprint/__init__.py @@ -4,6 +4,7 @@ DependsOnMixin, RoleBlueprint, AccountParameterBlueprint, + AggregationPolicyBlueprint, AlertBlueprint, BusinessRoleBlueprint, DatabaseBlueprint, @@ -26,6 +27,7 @@ PipeBlueprint, PrimaryKeyBlueprint, ProcedureBlueprint, + ProjectionPolicyBlueprint, ResourceMonitorBlueprint, RowAccessPolicyBlueprint, SchemaBlueprint, @@ -85,5 +87,13 @@ from .object_type import ObjectType from .permission_model import PermissionModel, PermissionModelCreateGrant, PermissionModelFutureGrant, PermissionModelRuleset -from .reference import ForeignKeyReference, IndexReference, MaskingPolicyReference, RowAccessPolicyReference, TagReference +from .reference import ( + AggregationPolicyReference, + ForeignKeyReference, + IndexReference, + MaskingPolicyReference, + ProjectionPolicyReference, + RowAccessPolicyReference, + TagReference, +) from .stage import StageWithPath, StageUploadFile diff --git a/snowddl/blueprint/blueprint.py b/snowddl/blueprint/blueprint.py index 616f13b..637e0e5 100644 --- a/snowddl/blueprint/blueprint.py +++ b/snowddl/blueprint/blueprint.py @@ -27,7 +27,15 @@ TableConstraintIdent, ) from .object_type import ObjectType -from .reference import ForeignKeyReference, IndexReference, MaskingPolicyReference, RowAccessPolicyReference, TagReference +from .reference import ( + AggregationPolicyReference, + ForeignKeyReference, + IndexReference, + MaskingPolicyReference, + ProjectionPolicyReference, + RowAccessPolicyReference, + TagReference, +) from .stage import StageWithPath from ..model import BaseModelWithConfig @@ -57,6 +65,11 @@ class AccountParameterBlueprint(AbstractBlueprint): value: Union[bool, float, int, str] +class AggregationPolicyBlueprint(SchemaObjectBlueprint): + body: str + references: List[AggregationPolicyReference] + + class AlertBlueprint(SchemaObjectBlueprint): full_name: SchemaObjectIdent warehouse: AccountObjectIdent @@ -200,6 +213,7 @@ class MaskingPolicyBlueprint(SchemaObjectBlueprint): arguments: List[NameWithType] returns: DataType body: str + exempt_other_policies: bool = False references: List[MaskingPolicyReference] @@ -258,6 +272,11 @@ class ProcedureBlueprint(SchemaObjectBlueprint): secrets: Optional[Dict[str, SchemaObjectIdent]] = None +class ProjectionPolicyBlueprint(SchemaObjectBlueprint): + body: str + references: List[ProjectionPolicyReference] + + class ResourceMonitorBlueprint(AbstractBlueprint): full_name: AccountObjectIdent credit_quota: int diff --git a/snowddl/blueprint/object_type.py b/snowddl/blueprint/object_type.py index cd10dd4..4b1cd3d 100644 --- a/snowddl/blueprint/object_type.py +++ b/snowddl/blueprint/object_type.py @@ -15,6 +15,12 @@ class ObjectType(Enum): "blueprint_cls": "AccountParameterBlueprint", } + AGGREGATION_POLICY = { + "singular": "AGGREGATION POLICY", + "plural": "AGGREGATION POLICIES", + "blueprint_cls": "AggregationPolicyBlueprint", + } + ALERT = { "singular": "ALERT", "plural": "ALERTS", @@ -142,6 +148,12 @@ class ObjectType(Enum): "blueprint_cls": "ProcedureBlueprint", } + PROJECTION_POLICY = { + "singular": "PROJECTION POLICY", + "plural": "PROJECTION POLICIES", + "blueprint_cls": "ProjectionPolicyBlueprint", + } + RESOURCE_MONITOR = { "singular": "RESOURCE MONITOR", "plural": "RESOURCE MONITORS", diff --git a/snowddl/blueprint/reference.py b/snowddl/blueprint/reference.py index 935615f..89053be 100644 --- a/snowddl/blueprint/reference.py +++ b/snowddl/blueprint/reference.py @@ -5,6 +5,12 @@ from ..model import BaseModelWithConfig +class AggregationPolicyReference(BaseModelWithConfig): + object_type: ObjectType + object_name: SchemaObjectIdent + columns: List[Ident] + + class ForeignKeyReference(BaseModelWithConfig): columns: List[Ident] ref_table_name: SchemaObjectIdent @@ -22,6 +28,12 @@ class MaskingPolicyReference(BaseModelWithConfig): columns: List[Ident] +class ProjectionPolicyReference(BaseModelWithConfig): + object_type: ObjectType + object_name: SchemaObjectIdent + column: Ident + + class RowAccessPolicyReference(BaseModelWithConfig): object_type: ObjectType object_name: SchemaObjectIdent diff --git a/snowddl/config.py b/snowddl/config.py index 50bf1dd..2ce2a97 100644 --- a/snowddl/config.py +++ b/snowddl/config.py @@ -79,7 +79,11 @@ def get_blueprints_by_type_and_pattern(self, cls: Type[T_Blueprint], pattern: st else: include_full_names.update(full_name for full_name in all_blueprints if regexp.match(full_name)) - return {full_name: bp for full_name, bp in all_blueprints.items() if full_name in include_full_names and full_name not in exclude_full_names} + return { + full_name: bp + for full_name, bp in all_blueprints.items() + if full_name in include_full_names and full_name not in exclude_full_names + } def get_placeholder(self, name: str) -> Union[bool, float, int, str]: if name not in self.placeholders: diff --git a/snowddl/parser/__init__.py b/snowddl/parser/__init__.py index c6f869d..f5679de 100644 --- a/snowddl/parser/__init__.py +++ b/snowddl/parser/__init__.py @@ -1,5 +1,6 @@ from ._parsed_file import ParsedFile from .account_params import AccountParameterParser +from .aggregation_policy import AggregationPolicyParser from .alert import AlertParser from .business_role import BusinessRoleParser from .database import DatabaseParser @@ -21,6 +22,7 @@ from .pipe import PipeParser from .placeholder import PlaceholderParser from .procedure import ProcedureParser +from .projection_policy import ProjectionPolicyParser from .resource_monitor import ResourceMonitorParser from .row_access_policy import RowAccessPolicyParser from .schema import SchemaParser @@ -63,7 +65,9 @@ ViewParser, PipeParser, TaskParser, + AggregationPolicyParser, MaskingPolicyParser, + ProjectionPolicyParser, RowAccessPolicyParser, OutboundShareParser, TechnicalRoleParser, @@ -94,7 +98,9 @@ ViewParser, PipeParser, TaskParser, + AggregationPolicyParser, MaskingPolicyParser, + ProjectionPolicyParser, RowAccessPolicyParser, AlertParser, ] diff --git a/snowddl/parser/aggregation_policy.py b/snowddl/parser/aggregation_policy.py new file mode 100644 index 0000000..e5b21fc --- /dev/null +++ b/snowddl/parser/aggregation_policy.py @@ -0,0 +1,76 @@ +from snowddl.blueprint import ( + AggregationPolicyBlueprint, + AggregationPolicyReference, + Ident, + ObjectType, + SchemaObjectIdent, + build_schema_object_ident, +) +from snowddl.parser.abc_parser import AbstractParser, ParsedFile + + +# fmt: off +aggregation_policy_json_schema = { + "type": "object", + "properties": { + "body": { + "type": "string" + }, + "references": { + "type": "array", + "items": { + "type": "object", + "properties": { + "object_type": { + "type": "string" + }, + "object_name": { + "type": "string" + }, + "columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + }, + "required": ["object_type", "object_name"], + "additionalProperties": False + }, + "minItems": 1 + }, + "comment": { + "type": "string" + } + }, + "required": ["body"], + "additionalProperties": False +} +# fmt: on + + +class AggregationPolicyParser(AbstractParser): + def load_blueprints(self): + self.parse_schema_object_files("aggregation_policy", aggregation_policy_json_schema, self.process_aggregation_policy) + + def process_aggregation_policy(self, f: ParsedFile): + references = [] + + for a in f.params.get("references", []): + ref = AggregationPolicyReference( + object_type=ObjectType[a["object_type"].upper()], + object_name=build_schema_object_ident(self.env_prefix, a["object_name"], f.database, f.schema), + columns=[Ident(c) for c in a.get("columns", [])], + ) + + references.append(ref) + + bp = AggregationPolicyBlueprint( + full_name=SchemaObjectIdent(self.env_prefix, f.database, f.schema, f.name), + body=f.params["body"], + references=references, + comment=f.params.get("comment"), + ) + + self.config.add_blueprint(bp) diff --git a/snowddl/parser/database.py b/snowddl/parser/database.py index fce61e7..7d4e785 100644 --- a/snowddl/parser/database.py +++ b/snowddl/parser/database.py @@ -64,8 +64,10 @@ def load_blueprints(self): database_name = database_path.name.upper() database_params = self.parse_single_file(database_path / "params.yaml", database_json_schema) + # fmt: off databases_permission_model_name = database_params.get("permission_model", self.config.DEFAULT_PERMISSION_MODEL).upper() database_permission_model = self.config.get_permission_model(databases_permission_model_name) + # fmt: on if not database_permission_model.ruleset.create_database_owner_role: for k in database_params: diff --git a/snowddl/parser/masking_policy.py b/snowddl/parser/masking_policy.py index 2a483fe..a83338c 100644 --- a/snowddl/parser/masking_policy.py +++ b/snowddl/parser/masking_policy.py @@ -27,6 +27,9 @@ "body": { "type": "string" }, + "exempt_other_policies": { + "type": "boolean" + }, "references": { "type": "array", "items": { @@ -83,6 +86,7 @@ def process_masking_policy(self, f: ParsedFile): body=f.params["body"], arguments=arguments, returns=DataType(f.params["returns"]), + exempt_other_policies=f.params.get("exempt_other_policies", False), references=references, comment=f.params.get("comment"), ) diff --git a/snowddl/parser/projection_policy.py b/snowddl/parser/projection_policy.py new file mode 100644 index 0000000..fac3c59 --- /dev/null +++ b/snowddl/parser/projection_policy.py @@ -0,0 +1,72 @@ +from snowddl.blueprint import ( + ProjectionPolicyBlueprint, + ProjectionPolicyReference, + Ident, + ObjectType, + SchemaObjectIdent, + build_schema_object_ident, +) +from snowddl.parser.abc_parser import AbstractParser, ParsedFile + + +# fmt: off +projection_policy_json_schema = { + "type": "object", + "properties": { + "body": { + "type": "string" + }, + "references": { + "type": "array", + "items": { + "type": "object", + "properties": { + "object_type": { + "type": "string" + }, + "object_name": { + "type": "string" + }, + "column": { + "type": "string" + } + }, + "required": ["object_type", "object_name", "column"], + "additionalProperties": False + }, + "minItems": 1 + }, + "comment": { + "type": "string" + } + }, + "required": ["body"], + "additionalProperties": False +} +# fmt: on + + +class ProjectionPolicyParser(AbstractParser): + def load_blueprints(self): + self.parse_schema_object_files("projection_policy", projection_policy_json_schema, self.process_projection_policy) + + def process_projection_policy(self, f: ParsedFile): + references = [] + + for a in f.params.get("references", []): + ref = ProjectionPolicyReference( + object_type=ObjectType[a["object_type"].upper()], + object_name=build_schema_object_ident(self.env_prefix, a["object_name"], f.database, f.schema), + column=Ident(a["column"]), + ) + + references.append(ref) + + bp = ProjectionPolicyBlueprint( + full_name=SchemaObjectIdent(self.env_prefix, f.database, f.schema, f.name), + body=f.params["body"], + references=references, + comment=f.params.get("comment"), + ) + + self.config.add_blueprint(bp) diff --git a/snowddl/parser/schema.py b/snowddl/parser/schema.py index eab3f78..b24dae2 100644 --- a/snowddl/parser/schema.py +++ b/snowddl/parser/schema.py @@ -91,8 +91,10 @@ def load_blueprints(self): database_name = database_path.name.upper() schema_name = schema_path.name.upper() + # fmt: off database_permission_model_name = database_params.get("permission_model", self.config.DEFAULT_PERMISSION_MODEL).upper() schema_permission_model_name = schema_params.get("permission_model", database_permission_model_name).upper() + # fmt: on database_permission_model = self.config.get_permission_model(database_permission_model_name) schema_permission_model = self.config.get_permission_model(schema_permission_model_name) diff --git a/snowddl/resolver/__init__.py b/snowddl/resolver/__init__.py index 5c1cf19..54fab32 100644 --- a/snowddl/resolver/__init__.py +++ b/snowddl/resolver/__init__.py @@ -2,6 +2,7 @@ from .abc_role_resolver import AbstractRoleResolver from .abc_schema_object_resolver import AbstractSchemaObjectResolver from .account_params import AccountParameterResolver +from .aggregation_policy import AggregationPolicyResolver from .alert import AlertResolver from .business_role import BusinessRoleResolver from .clone_table import CloneTableResolver @@ -26,6 +27,7 @@ from .pipe import PipeResolver from .primary_key import PrimaryKeyResolver from .procedure import ProcedureResolver +from .projection_policy import ProjectionPolicyResolver from .resource_monitor import ResourceMonitorResolver from .row_access_policy import RowAccessPolicyResolver from .sequence import SequenceResolver @@ -82,7 +84,9 @@ ViewResolver, PipeResolver, TaskResolver, + AggregationPolicyResolver, MaskingPolicyResolver, + ProjectionPolicyResolver, RowAccessPolicyResolver, OutboundShareResolver, TechnicalRoleResolver, @@ -118,7 +122,9 @@ ViewResolver, PipeResolver, TaskResolver, + AggregationPolicyResolver, MaskingPolicyResolver, + ProjectionPolicyResolver, RowAccessPolicyResolver, AlertResolver, ] diff --git a/snowddl/resolver/aggregation_policy.py b/snowddl/resolver/aggregation_policy.py new file mode 100644 index 0000000..2f0c767 --- /dev/null +++ b/snowddl/resolver/aggregation_policy.py @@ -0,0 +1,227 @@ +from json import loads + +from snowddl.blueprint import AggregationPolicyBlueprint, ObjectType, Edition, SchemaObjectIdent +from snowddl.resolver.abc_schema_object_resolver import AbstractSchemaObjectResolver, ResolveResult + + +class AggregationPolicyResolver(AbstractSchemaObjectResolver): + skip_on_empty_blueprints = True + skip_min_edition = Edition.ENTERPRISE + + def get_object_type(self) -> ObjectType: + return ObjectType.AGGREGATION_POLICY + + def get_existing_objects_in_schema(self, schema: dict): + existing_objects = {} + + cur = self.engine.execute_meta( + "SHOW AGGREGATION POLICIES IN SCHEMA {database:i}.{schema:i}", + { + "database": schema["database"], + "schema": schema["schema"], + }, + ) + + for r in cur: + full_name = f"{r['database_name']}.{r['schema_name']}.{r['name']}" + + existing_objects[full_name] = { + "database": r["database_name"], + "schema": r["schema_name"], + "name": r["name"], + "comment": r["comment"] if r["comment"] else None, + } + + return existing_objects + + def get_blueprints(self): + return self.config.get_blueprints_by_type(AggregationPolicyBlueprint) + + def create_object(self, bp: AggregationPolicyBlueprint): + self._create_policy(bp) + self._apply_policy_refs(bp, skip_existing=True) + + return ResolveResult.CREATE + + def compare_object(self, bp: AggregationPolicyBlueprint, row: dict): + cur = self.engine.execute_meta( + "DESC AGGREGATION POLICY {full_name:i}", + { + "full_name": bp.full_name, + }, + ) + + r = cur.fetchone() + result = ResolveResult.NOCHANGE + + if self._apply_policy_refs(bp): + result = ResolveResult.ALTER + + if r["body"] != bp.body: + self.engine.execute_unsafe_ddl( + "ALTER AGGREGATION POLICY {full_name:i} SET BODY -> {body:r}", + { + "full_name": bp.full_name, + "body": bp.body, + }, + condition=self.engine.settings.execute_aggregation_policy, + ) + + result = ResolveResult.ALTER + + if row["comment"] != bp.comment: + self.engine.execute_unsafe_ddl( + "ALTER AGGREGATION POLICY {full_name:i} SET COMMENT = {comment}", + { + "full_name": bp.full_name, + "comment": bp.comment, + }, + condition=self.engine.settings.execute_aggregation_policy, + ) + + result = ResolveResult.ALTER + + return result + + def drop_object(self, row: dict): + self._drop_policy_refs(SchemaObjectIdent("", row["database"], row["schema"], row["name"])) + self._drop_policy(SchemaObjectIdent("", row["database"], row["schema"], row["name"])) + + return ResolveResult.DROP + + def _create_policy(self, bp: AggregationPolicyBlueprint): + query = self.engine.query_builder() + + query.append( + "CREATE AGGREGATION POLICY {full_name:i}", + { + "full_name": bp.full_name, + }, + ) + + query.append_nl( + "AS () RETURNS AGGREGATION_CONSTRAINT -> {body:r}", + { + "body": bp.body, + }, + ) + + if bp.comment: + query.append_nl( + "COMMENT = {comment}", + { + "comment": bp.comment, + }, + ) + + self.engine.execute_unsafe_ddl(query, condition=self.engine.settings.execute_aggregation_policy) + + def _drop_policy(self, policy: SchemaObjectIdent): + self.engine.execute_unsafe_ddl( + "DROP AGGREGATION POLICY {full_name:i}", + {"full_name": policy}, + condition=self.engine.settings.execute_aggregation_policy, + ) + + def _apply_policy_refs(self, bp: AggregationPolicyBlueprint, skip_existing=False): + existing_policy_refs = {} if skip_existing else self._get_existing_policy_refs(bp.full_name) + applied_change = False + + for ref in bp.references: + ref_key = f"{ref.object_type.name}|{ref.object_name}|{bp.full_name}" + + # Policy was applied before + if ref_key in existing_policy_refs: + existing_ref_columns = existing_policy_refs[ref_key]["columns"] + del existing_policy_refs[ref_key] + + # Policy was applied to exactly the same columns + if [str(c) for c in ref.columns] == existing_ref_columns: + continue + + # Apply new policy or replace existing policy with different columns + query = self.engine.query_builder() + + query.append( + "ALTER {object_type:r} {object_name:i} SET AGGREGATION POLICY {policy_name:i}", + { + "object_type": ref.object_type.simplified, + "object_name": ref.object_name, + "policy_name": bp.full_name, + }, + ) + + if ref.columns: + query.append( + "ENTITY KEY ({columns:i})", + { + "columns": ref.columns, + }, + ) + + query.append("FORCE") + + self.engine.execute_unsafe_ddl(query, condition=self.engine.settings.execute_aggregation_policy) + applied_change = True + + # Remove remaining policy references which no longer exist in blueprint + for existing_ref in existing_policy_refs.values(): + # TODO: consider use case when object switches to another aggregation policy resolved in parallel + self.engine.execute_unsafe_ddl( + "ALTER {object_type:r} {database:i}.{schema:i}.{name:i} UNSET AGGREGATION POLICY", + { + "object_type": existing_ref["object_type"], + "database": existing_ref["database"], + "schema": existing_ref["schema"], + "name": existing_ref["name"], + }, + condition=self.engine.settings.execute_aggregation_policy, + ) + + applied_change = True + + return applied_change + + def _drop_policy_refs(self, policy_name: SchemaObjectIdent): + existing_policy_refs = self._get_existing_policy_refs(policy_name) + + for existing_ref in existing_policy_refs.values(): + # TODO: consider use case when object switches to another aggregation policy resolved in parallel + self.engine.execute_unsafe_ddl( + "ALTER {object_type:r} {database:i}.{schema:i}.{name:i} UNSET AGGREGATION POLICY", + { + "object_type": existing_ref["object_type"], + "database": existing_ref["database"], + "schema": existing_ref["schema"], + "name": existing_ref["name"], + }, + condition=self.engine.settings.execute_aggregation_policy, + ) + + def _get_existing_policy_refs(self, policy_name: SchemaObjectIdent): + existing_policy_refs = {} + + cur = self.engine.execute_meta( + "SELECT * FROM TABLE({database:i}.information_schema.policy_references(policy_name => {policy_name}))", + { + "database": policy_name.database_full_name, + "policy_name": policy_name, + }, + ) + + for r in cur: + ref_key = ( + f"{r['REF_ENTITY_DOMAIN']}" + f"|{r['REF_DATABASE_NAME']}.{r['REF_SCHEMA_NAME']}.{r['REF_ENTITY_NAME']}" + f"|{r['POLICY_DB']}.{r['POLICY_SCHEMA']}.{r['POLICY_NAME']}" + ) + + existing_policy_refs[ref_key] = { + "object_type": r["REF_ENTITY_DOMAIN"], + "database": r["REF_DATABASE_NAME"], + "schema": r["REF_SCHEMA_NAME"], + "name": r["REF_ENTITY_NAME"], + "columns": loads(r["REF_ARG_COLUMN_NAMES"]) if r["REF_ARG_COLUMN_NAMES"] else [], + } + + return existing_policy_refs diff --git a/snowddl/resolver/masking_policy.py b/snowddl/resolver/masking_policy.py index a162a8a..53f73bd 100644 --- a/snowddl/resolver/masking_policy.py +++ b/snowddl/resolver/masking_policy.py @@ -1,3 +1,5 @@ +from json import loads + from snowddl.blueprint import MaskingPolicyBlueprint, ObjectType, Edition, SchemaObjectIdent from snowddl.resolver.abc_schema_object_resolver import AbstractSchemaObjectResolver, ResolveResult @@ -27,6 +29,7 @@ def get_existing_objects_in_schema(self, schema: dict): "database": r["database_name"], "schema": r["schema_name"], "name": r["name"], + "options": loads(r["options"]) if r["options"] else {}, "comment": r["comment"] if r["comment"] else None, } @@ -52,9 +55,11 @@ def compare_object(self, bp: MaskingPolicyBlueprint, row: dict): r = cur.fetchone() # If signature or return type was changed, policy and all references must be dropped and created again - if r["signature"] != f"({', '.join([f'{a.name} {a.type.base_type.name}' for a in bp.arguments])})" or r[ - "return_type" - ] != str(bp.returns): + if ( + r["signature"] != f"({', '.join([f'{a.name} {a.type.base_type.name}' for a in bp.arguments])})" + or r["return_type"] != str(bp.returns) + or row["options"].get("EXEMPT_OTHER_POLICIES", False) != bp.exempt_other_policies + ): self._drop_policy_refs(bp.full_name) self._drop_policy(bp.full_name) @@ -136,6 +141,9 @@ def _create_policy(self, bp: MaskingPolicyBlueprint): }, ) + if bp.exempt_other_policies: + query.append_nl("EXEMPT_OTHER_POLICIES = TRUE") + if bp.comment: query.append_nl( "COMMENT = {comment}", diff --git a/snowddl/resolver/projection_policy.py b/snowddl/resolver/projection_policy.py new file mode 100644 index 0000000..50405f8 --- /dev/null +++ b/snowddl/resolver/projection_policy.py @@ -0,0 +1,212 @@ +from snowddl.blueprint import ProjectionPolicyBlueprint, ObjectType, Edition, SchemaObjectIdent +from snowddl.resolver.abc_schema_object_resolver import AbstractSchemaObjectResolver, ResolveResult + + +class ProjectionPolicyResolver(AbstractSchemaObjectResolver): + skip_on_empty_blueprints = True + skip_min_edition = Edition.ENTERPRISE + + def get_object_type(self) -> ObjectType: + return ObjectType.PROJECTION_POLICY + + def get_existing_objects_in_schema(self, schema: dict): + existing_objects = {} + + cur = self.engine.execute_meta( + "SHOW PROJECTION POLICIES IN SCHEMA {database:i}.{schema:i}", + { + "database": schema["database"], + "schema": schema["schema"], + }, + ) + + for r in cur: + full_name = f"{r['database_name']}.{r['schema_name']}.{r['name']}" + + existing_objects[full_name] = { + "database": r["database_name"], + "schema": r["schema_name"], + "name": r["name"], + "comment": r["comment"] if r["comment"] else None, + } + + return existing_objects + + def get_blueprints(self): + return self.config.get_blueprints_by_type(ProjectionPolicyBlueprint) + + def create_object(self, bp: ProjectionPolicyBlueprint): + self._create_policy(bp) + self._apply_policy_refs(bp, skip_existing=True) + + return ResolveResult.CREATE + + def compare_object(self, bp: ProjectionPolicyBlueprint, row: dict): + cur = self.engine.execute_meta( + "DESC PROJECTION POLICY {full_name:i}", + { + "full_name": bp.full_name, + }, + ) + + r = cur.fetchone() + result = ResolveResult.NOCHANGE + + if self._apply_policy_refs(bp): + result = ResolveResult.ALTER + + if r["body"] != bp.body: + self.engine.execute_unsafe_ddl( + "ALTER PROJECTION POLICY {full_name:i} SET BODY -> {body:r}", + { + "full_name": bp.full_name, + "body": bp.body, + }, + condition=self.engine.settings.execute_projection_policy, + ) + + result = ResolveResult.ALTER + + if row["comment"] != bp.comment: + self.engine.execute_unsafe_ddl( + "ALTER PROJECTION POLICY {full_name:i} SET COMMENT = {comment}", + { + "full_name": bp.full_name, + "comment": bp.comment, + }, + condition=self.engine.settings.execute_projection_policy, + ) + + result = ResolveResult.ALTER + + return result + + def drop_object(self, row: dict): + self._drop_policy_refs(SchemaObjectIdent("", row["database"], row["schema"], row["name"])) + self._drop_policy(SchemaObjectIdent("", row["database"], row["schema"], row["name"])) + + return ResolveResult.DROP + + def _create_policy(self, bp: ProjectionPolicyBlueprint): + query = self.engine.query_builder() + + query.append( + "CREATE PROJECTION POLICY {full_name:i}", + { + "full_name": bp.full_name, + }, + ) + + query.append_nl( + "AS () RETURNS PROJECTION_CONSTRAINT -> {body:r}", + { + "body": bp.body, + }, + ) + + if bp.comment: + query.append_nl( + "COMMENT = {comment}", + { + "comment": bp.comment, + }, + ) + + self.engine.execute_unsafe_ddl(query, condition=self.engine.settings.execute_projection_policy) + + def _drop_policy(self, policy: SchemaObjectIdent): + self.engine.execute_unsafe_ddl( + "DROP PROJECTION POLICY {full_name:i}", + {"full_name": policy}, + condition=self.engine.settings.execute_projection_policy, + ) + + def _apply_policy_refs(self, bp: ProjectionPolicyBlueprint, skip_existing=False): + existing_policy_refs = {} if skip_existing else self._get_existing_policy_refs(bp.full_name) + applied_change = False + + for ref in bp.references: + ref_key = f"{ref.object_type.name}|{ref.object_name}|{ref.column}" + + # Policy was applied before + if ref_key in existing_policy_refs: + del existing_policy_refs[ref_key] + continue + + # Apply new policy or replace existing policy + self.engine.execute_unsafe_ddl( + "ALTER {object_type:r} {object_name:i} MODIFY COLUMN {column:i} SET PROJECTION POLICY {policy_name:i} FORCE", + { + "object_type": ref.object_type.simplified, + "object_name": ref.object_name, + "column": ref.column, + "policy_name": bp.full_name, + }, + condition=self.engine.settings.execute_projection_policy, + ) + + applied_change = True + + # Remove remaining policy references which no longer exist in blueprint + for existing_ref in existing_policy_refs.values(): + # TODO: consider use case when object switches to another projection policy resolved in parallel + self.engine.execute_unsafe_ddl( + "ALTER {object_type:r} {database:i}.{schema:i}.{name:i} MODIFY COLUMN {column:i} UNSET PROJECTION POLICY", + { + "object_type": existing_ref["object_type"], + "database": existing_ref["database"], + "schema": existing_ref["schema"], + "column": existing_ref["column"], + "name": existing_ref["name"], + }, + condition=self.engine.settings.execute_projection_policy, + ) + + applied_change = True + + return applied_change + + def _drop_policy_refs(self, policy_name: SchemaObjectIdent): + existing_policy_refs = self._get_existing_policy_refs(policy_name) + + for existing_ref in existing_policy_refs.values(): + # TODO: consider use case when object switches to another projection policy resolved in parallel + self.engine.execute_unsafe_ddl( + "ALTER {object_type:r} {database:i}.{schema:i}.{name:i} MODIFY COLUMN {column:i} UNSET PROJECTION POLICY", + { + "object_type": existing_ref["object_type"], + "database": existing_ref["database"], + "schema": existing_ref["schema"], + "column": existing_ref["column"], + "name": existing_ref["name"], + }, + condition=self.engine.settings.execute_projection_policy, + ) + + def _get_existing_policy_refs(self, policy_name: SchemaObjectIdent): + existing_policy_refs = {} + + cur = self.engine.execute_meta( + "SELECT * FROM TABLE({database:i}.information_schema.policy_references(policy_name => {policy_name}))", + { + "database": policy_name.database_full_name, + "policy_name": policy_name, + }, + ) + + for r in cur: + ref_key = ( + f"{r['REF_ENTITY_DOMAIN']}" + f"|{r['REF_DATABASE_NAME']}.{r['REF_SCHEMA_NAME']}.{r['REF_ENTITY_NAME']}" + f"|{r['REF_COLUMN_NAME']}" + ) + + existing_policy_refs[ref_key] = { + "object_type": r["REF_ENTITY_DOMAIN"], + "database": r["REF_DATABASE_NAME"], + "schema": r["REF_SCHEMA_NAME"], + "name": r["REF_ENTITY_NAME"], + "column": r["REF_COLUMN_NAME"], + } + + return existing_policy_refs diff --git a/snowddl/settings.py b/snowddl/settings.py index dbe5ef7..8de9b1f 100644 --- a/snowddl/settings.py +++ b/snowddl/settings.py @@ -9,7 +9,9 @@ class SnowDDLSettings(BaseModelWithConfig): execute_safe_ddl: bool = False execute_unsafe_ddl: bool = False execute_replace_table: bool = False + execute_aggregation_policy: bool = False execute_masking_policy: bool = False + execute_projection_policy: bool = False execute_row_access_policy: bool = False execute_account_params: bool = False execute_network_policy: bool = False diff --git a/snowddl/version.py b/snowddl/version.py index cc09a06..f634f5b 100644 --- a/snowddl/version.py +++ b/snowddl/version.py @@ -1 +1 @@ -__version__ = "0.28.3" +__version__ = "0.29.0" diff --git a/test/_config/step1/db1/sc1/aggregation_policy/ap001_ap1.yaml b/test/_config/step1/db1/sc1/aggregation_policy/ap001_ap1.yaml new file mode 100644 index 0000000..a3d0ab3 --- /dev/null +++ b/test/_config/step1/db1/sc1/aggregation_policy/ap001_ap1.yaml @@ -0,0 +1,13 @@ +body: |- + CASE WHEN IS_ROLE_IN_SESSION('SYSADMIN') THEN NO_AGGREGATION_CONSTRAINT() + ELSE AGGREGATION_CONSTRAINT(MIN_GROUP_SIZE => 5) + END + +references: + - object_type: TABLE + object_name: ap001_tb1 + + - object_type: VIEW + object_name: ap001_vw1 + +comment: abc diff --git a/test/_config/step1/db1/sc1/masking_policy/mp001_mp1.yaml b/test/_config/step1/db1/sc1/masking_policy/mp001_mp1.yaml new file mode 100644 index 0000000..6c297f4 --- /dev/null +++ b/test/_config/step1/db1/sc1/masking_policy/mp001_mp1.yaml @@ -0,0 +1,21 @@ +arguments: + name: VARCHAR(255) + id: NUMBER(38,0) + +returns: VARCHAR(255) + +body: |- + CASE WHEN id > 100 THEN name + ELSE '*****' + END + +references: + - object_type: TABLE + object_name: mp001_tb1 + columns: [name, id] + + - object_type: VIEW + object_name: mp001_vw1 + columns: [name, id] + +comment: abc diff --git a/test/_config/step1/db1/sc1/projection_policy/pp001_pp1.yaml b/test/_config/step1/db1/sc1/projection_policy/pp001_pp1.yaml new file mode 100644 index 0000000..5c7be96 --- /dev/null +++ b/test/_config/step1/db1/sc1/projection_policy/pp001_pp1.yaml @@ -0,0 +1,19 @@ +body: |- + CASE WHEN IS_ROLE_IN_SESSION('SYSADMIN') THEN PROJECTION_CONSTRAINT(ALLOW => true) + ELSE PROJECTION_CONSTRAINT(ALLOW => false) + END + +references: + - object_type: TABLE + object_name: pp001_tb1 + column: id + + - object_type: TABLE + object_name: pp001_tb1 + column: name + + - object_type: VIEW + object_name: pp001_vw1 + column: id + +comment: abc diff --git a/test/_config/step1/db1/sc1/row_access_policy/rp001_rp1.yaml b/test/_config/step1/db1/sc1/row_access_policy/rp001_rp1.yaml new file mode 100644 index 0000000..e72617b --- /dev/null +++ b/test/_config/step1/db1/sc1/row_access_policy/rp001_rp1.yaml @@ -0,0 +1,16 @@ +arguments: + name: VARCHAR(255) + +body: |- + name LIKE '%John%' + +references: + - object_type: TABLE + object_name: rp001_tb1 + columns: [name] + + - object_type: VIEW + object_name: rp001_vw1 + columns: [name] + +comment: abc diff --git a/test/_config/step1/db1/sc1/table/ap001_tb1.yaml b/test/_config/step1/db1/sc1/table/ap001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step1/db1/sc1/table/ap001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step1/db1/sc1/table/mp001_tb1.yaml b/test/_config/step1/db1/sc1/table/mp001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step1/db1/sc1/table/mp001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step1/db1/sc1/table/pp001_tb1.yaml b/test/_config/step1/db1/sc1/table/pp001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step1/db1/sc1/table/pp001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step1/db1/sc1/table/rp001_tb1.yaml b/test/_config/step1/db1/sc1/table/rp001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step1/db1/sc1/table/rp001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step1/db1/sc1/view/ap001_vw1.yaml b/test/_config/step1/db1/sc1/view/ap001_vw1.yaml new file mode 100644 index 0000000..e99ffb9 --- /dev/null +++ b/test/_config/step1/db1/sc1/view/ap001_vw1.yaml @@ -0,0 +1,3 @@ +text: |- + SELECT * FROM ap001_tb1 + diff --git a/test/_config/step1/db1/sc1/view/mp001_vw1.yaml b/test/_config/step1/db1/sc1/view/mp001_vw1.yaml new file mode 100644 index 0000000..56f642d --- /dev/null +++ b/test/_config/step1/db1/sc1/view/mp001_vw1.yaml @@ -0,0 +1,2 @@ +text: |- + SELECT * FROM mp001_tb1 diff --git a/test/_config/step1/db1/sc1/view/pp001_vw1.yaml b/test/_config/step1/db1/sc1/view/pp001_vw1.yaml new file mode 100644 index 0000000..c668abd --- /dev/null +++ b/test/_config/step1/db1/sc1/view/pp001_vw1.yaml @@ -0,0 +1,2 @@ +text: |- + SELECT * FROM pp001_tb1 diff --git a/test/_config/step1/db1/sc1/view/rp001_vw1.yaml b/test/_config/step1/db1/sc1/view/rp001_vw1.yaml new file mode 100644 index 0000000..7e46459 --- /dev/null +++ b/test/_config/step1/db1/sc1/view/rp001_vw1.yaml @@ -0,0 +1,3 @@ +text: |- + SELECT * FROM rp001_tb1 + diff --git a/test/_config/step2/db1/sc1/aggregation_policy/ap001_ap1.yaml b/test/_config/step2/db1/sc1/aggregation_policy/ap001_ap1.yaml new file mode 100644 index 0000000..3e966f8 --- /dev/null +++ b/test/_config/step2/db1/sc1/aggregation_policy/ap001_ap1.yaml @@ -0,0 +1,15 @@ +body: |- + CASE WHEN IS_ROLE_IN_SESSION('SYSADMIN') THEN NO_AGGREGATION_CONSTRAINT() + ELSE AGGREGATION_CONSTRAINT(MIN_ROW_COUNT => 5, MIN_ENTITY_COUNT => 2) + END + +references: + - object_type: TABLE + object_name: ap001_tb1 + columns: [id] + + - object_type: VIEW + object_name: ap001_vw1 + columns: [id] + +comment: cde diff --git a/test/_config/step2/db1/sc1/masking_policy/mp001_mp1.yaml b/test/_config/step2/db1/sc1/masking_policy/mp001_mp1.yaml new file mode 100644 index 0000000..7ad5221 --- /dev/null +++ b/test/_config/step2/db1/sc1/masking_policy/mp001_mp1.yaml @@ -0,0 +1,23 @@ +arguments: + name: VARCHAR(255) + id: NUMBER(38,0) + +returns: VARCHAR(255) + +body: |- + CASE WHEN id > 100 THEN name + ELSE '*****' + END + +exempt_other_policies: true + +references: + - object_type: TABLE + object_name: mp001_tb1 + columns: [name, id] + + - object_type: VIEW + object_name: mp001_vw1 + columns: [name, id] + +comment: cde diff --git a/test/_config/step2/db1/sc1/projection_policy/pp001_pp1.yaml b/test/_config/step2/db1/sc1/projection_policy/pp001_pp1.yaml new file mode 100644 index 0000000..08bfc1d --- /dev/null +++ b/test/_config/step2/db1/sc1/projection_policy/pp001_pp1.yaml @@ -0,0 +1,15 @@ +body: |- + CASE WHEN IS_ROLE_IN_SESSION('SYSADMIN') THEN PROJECTION_CONSTRAINT(ALLOW => true) + ELSE PROJECTION_CONSTRAINT(ALLOW => true) + END + +references: + - object_type: TABLE + object_name: pp001_tb1 + column: id + + - object_type: VIEW + object_name: pp001_vw1 + column: id + +comment: cde diff --git a/test/_config/step2/db1/sc1/row_access_policy/rp001_rp1.yaml b/test/_config/step2/db1/sc1/row_access_policy/rp001_rp1.yaml new file mode 100644 index 0000000..fbc4493 --- /dev/null +++ b/test/_config/step2/db1/sc1/row_access_policy/rp001_rp1.yaml @@ -0,0 +1,12 @@ +arguments: + name: VARCHAR(255) + +body: |- + name LIKE '%Gill%' + +references: + - object_type: TABLE + object_name: rp001_tb1 + columns: [name] + +comment: cde diff --git a/test/_config/step2/db1/sc1/table/ap001_tb1.yaml b/test/_config/step2/db1/sc1/table/ap001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step2/db1/sc1/table/ap001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step2/db1/sc1/table/mp001_tb1.yaml b/test/_config/step2/db1/sc1/table/mp001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step2/db1/sc1/table/mp001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step2/db1/sc1/table/pp001_tb1.yaml b/test/_config/step2/db1/sc1/table/pp001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step2/db1/sc1/table/pp001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step2/db1/sc1/table/rp001_tb1.yaml b/test/_config/step2/db1/sc1/table/rp001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step2/db1/sc1/table/rp001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step2/db1/sc1/view/ap001_vw1.yaml b/test/_config/step2/db1/sc1/view/ap001_vw1.yaml new file mode 100644 index 0000000..e99ffb9 --- /dev/null +++ b/test/_config/step2/db1/sc1/view/ap001_vw1.yaml @@ -0,0 +1,3 @@ +text: |- + SELECT * FROM ap001_tb1 + diff --git a/test/_config/step2/db1/sc1/view/mp001_vw1.yaml b/test/_config/step2/db1/sc1/view/mp001_vw1.yaml new file mode 100644 index 0000000..56f642d --- /dev/null +++ b/test/_config/step2/db1/sc1/view/mp001_vw1.yaml @@ -0,0 +1,2 @@ +text: |- + SELECT * FROM mp001_tb1 diff --git a/test/_config/step2/db1/sc1/view/pp001_vw1.yaml b/test/_config/step2/db1/sc1/view/pp001_vw1.yaml new file mode 100644 index 0000000..c668abd --- /dev/null +++ b/test/_config/step2/db1/sc1/view/pp001_vw1.yaml @@ -0,0 +1,2 @@ +text: |- + SELECT * FROM pp001_tb1 diff --git a/test/_config/step2/db1/sc1/view/rp001_vw1.yaml b/test/_config/step2/db1/sc1/view/rp001_vw1.yaml new file mode 100644 index 0000000..7e46459 --- /dev/null +++ b/test/_config/step2/db1/sc1/view/rp001_vw1.yaml @@ -0,0 +1,3 @@ +text: |- + SELECT * FROM rp001_tb1 + diff --git a/test/_config/step3/db1/sc1/aggregation_policy/ap001_ap2.yaml b/test/_config/step3/db1/sc1/aggregation_policy/ap001_ap2.yaml new file mode 100644 index 0000000..059055f --- /dev/null +++ b/test/_config/step3/db1/sc1/aggregation_policy/ap001_ap2.yaml @@ -0,0 +1,2 @@ +body: |- + NO_AGGREGATION_CONSTRAINT() diff --git a/test/_config/step3/db1/sc1/masking_policy/mp001_mp2.yaml b/test/_config/step3/db1/sc1/masking_policy/mp001_mp2.yaml new file mode 100644 index 0000000..559c5e1 --- /dev/null +++ b/test/_config/step3/db1/sc1/masking_policy/mp001_mp2.yaml @@ -0,0 +1,7 @@ +arguments: + name: VARCHAR(255) + +returns: VARCHAR(255) + +body: |- + name diff --git a/test/_config/step3/db1/sc1/projection_policy/pp001_pp2.yaml b/test/_config/step3/db1/sc1/projection_policy/pp001_pp2.yaml new file mode 100644 index 0000000..9e93b3b --- /dev/null +++ b/test/_config/step3/db1/sc1/projection_policy/pp001_pp2.yaml @@ -0,0 +1,2 @@ +body: |- + PROJECTION_CONSTRAINT(ALLOW => true) diff --git a/test/_config/step3/db1/sc1/row_access_policy/rp001_rp2.yaml b/test/_config/step3/db1/sc1/row_access_policy/rp001_rp2.yaml new file mode 100644 index 0000000..3ba0379 --- /dev/null +++ b/test/_config/step3/db1/sc1/row_access_policy/rp001_rp2.yaml @@ -0,0 +1,5 @@ +arguments: + name: VARCHAR(255) + +body: |- + name LIKE '%Gill%' diff --git a/test/_config/step3/db1/sc1/table/ap001_tb1.yaml b/test/_config/step3/db1/sc1/table/ap001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step3/db1/sc1/table/ap001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step3/db1/sc1/table/mp001_tb1.yaml b/test/_config/step3/db1/sc1/table/mp001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step3/db1/sc1/table/mp001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step3/db1/sc1/table/pp001_tb1.yaml b/test/_config/step3/db1/sc1/table/pp001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step3/db1/sc1/table/pp001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step3/db1/sc1/table/rp001_tb1.yaml b/test/_config/step3/db1/sc1/table/rp001_tb1.yaml new file mode 100644 index 0000000..c529f34 --- /dev/null +++ b/test/_config/step3/db1/sc1/table/rp001_tb1.yaml @@ -0,0 +1,3 @@ +columns: + id: NUMBER(38,0) + name: VARCHAR(255) diff --git a/test/_config/step3/db1/sc1/view/ap001_vw1.yaml b/test/_config/step3/db1/sc1/view/ap001_vw1.yaml new file mode 100644 index 0000000..e99ffb9 --- /dev/null +++ b/test/_config/step3/db1/sc1/view/ap001_vw1.yaml @@ -0,0 +1,3 @@ +text: |- + SELECT * FROM ap001_tb1 + diff --git a/test/_config/step3/db1/sc1/view/mp001_vw1.yaml b/test/_config/step3/db1/sc1/view/mp001_vw1.yaml new file mode 100644 index 0000000..56f642d --- /dev/null +++ b/test/_config/step3/db1/sc1/view/mp001_vw1.yaml @@ -0,0 +1,2 @@ +text: |- + SELECT * FROM mp001_tb1 diff --git a/test/_config/step3/db1/sc1/view/pp001_vw1.yaml b/test/_config/step3/db1/sc1/view/pp001_vw1.yaml new file mode 100644 index 0000000..c668abd --- /dev/null +++ b/test/_config/step3/db1/sc1/view/pp001_vw1.yaml @@ -0,0 +1,2 @@ +text: |- + SELECT * FROM pp001_tb1 diff --git a/test/_config/step3/db1/sc1/view/rp001_vw1.yaml b/test/_config/step3/db1/sc1/view/rp001_vw1.yaml new file mode 100644 index 0000000..7e46459 --- /dev/null +++ b/test/_config/step3/db1/sc1/view/rp001_vw1.yaml @@ -0,0 +1,3 @@ +text: |- + SELECT * FROM rp001_tb1 + diff --git a/test/run_test.sh b/test/run_test.sh index 95bb1fb..8f078c1 100755 --- a/test/run_test.sh +++ b/test/run_test.sh @@ -10,22 +10,22 @@ cd "${0%/*}" # Cleanup before -snowddl -c _config/step1 --apply-unsafe --apply-resource-monitor --apply-network-policy destroy +snowddl -c _config/step1 --apply-unsafe --apply-resource-monitor --apply-all-policy destroy # Apply step1 -snowddl -c _config/step1 --apply-unsafe --apply-resource-monitor --apply-network-policy apply +snowddl -c _config/step1 --apply-unsafe --apply-resource-monitor --apply-all-policy apply # Run test step1 pytest -k "step1" --tb=short */*.py # Apply step2 -snowddl -c _config/step2 --apply-unsafe --apply-replace-table --apply-resource-monitor --apply-network-policy --refresh-stage-encryption --refresh-secrets apply +snowddl -c _config/step2 --apply-unsafe --apply-replace-table --apply-resource-monitor --apply-all-policy --refresh-stage-encryption --refresh-secrets apply # Run test step2 pytest -k "step2" --tb=short */*.py # Apply step3 -snowddl -c _config/step3 --apply-unsafe --apply-replace-table --apply-resource-monitor --apply-network-policy --refresh-stage-encryption --refresh-secrets apply +snowddl -c _config/step3 --apply-unsafe --apply-replace-table --apply-resource-monitor --apply-all-policy --refresh-stage-encryption --refresh-secrets apply # Run test step3 pytest -k "step3" --tb=short */*.py