From a8a4a5725c685e1401506182a92de311c5dfdf74 Mon Sep 17 00:00:00 2001 From: "Valentin J. Hauner" Date: Tue, 10 Jan 2023 14:08:46 +0100 Subject: [PATCH 1/5] Adds support for lifecycle rules --- README.md | 4 ++ .../templates/objectstorage.yaml | 15 ++++++++ hybridcloud/backends/azureblob.py | 38 +++++++++++++++++-- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a87867e..ba6c3e2 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,10 @@ spec: retentionPeriodInDays: 1 # Days to keep deleted data, optional backup: enabled: false # Override the default backup strategy configured in the global operator config + lifecycle: # Define rules which determine the lifecycle of blob resources + rules: + - blobPrefix: foobar + deleteDaysAfterModification: 30 containers: # Only relevant for azure, list of containers to create in the bucket, for azure at least one is required, containers not on the list will be removed from the storage account, including their data - name: assets # Name of the container, required anonymousAccess: false # If set to true objects in the container can be accessed without authentication/authorization, only relevant if `security.anonymousAccess` is set to true, optional diff --git a/helm/hybrid-cloud-object-storage-operator-crds/templates/objectstorage.yaml b/helm/hybrid-cloud-object-storage-operator-crds/templates/objectstorage.yaml index 2c799d3..5463064 100644 --- a/helm/hybrid-cloud-object-storage-operator-crds/templates/objectstorage.yaml +++ b/helm/hybrid-cloud-object-storage-operator-crds/templates/objectstorage.yaml @@ -102,6 +102,21 @@ spec: properties: enabled: type: boolean + lifecycle: + type: object + properties: + rules: + type: array + items: + type: object + properties: + blobPrefix: + type: string + deleteDaysAfterModification: + type: number + required: + - blobPrefix + - deleteDaysAfterModification containers: type: array items: diff --git a/hybridcloud/backends/azureblob.py b/hybridcloud/backends/azureblob.py index c68b4fb..7381788 100644 --- a/hybridcloud/backends/azureblob.py +++ b/hybridcloud/backends/azureblob.py @@ -4,7 +4,9 @@ BlobServiceProperties, \ CorsRules, CorsRule, NetworkRuleSet, IPRule, VirtualNetworkRule, BlobContainer, \ StorageAccountCheckNameAvailabilityParameters, StorageAccountRegenerateKeyParameters, \ - DeleteRetentionPolicy, RestorePolicyProperties, ChangeFeed, LocalUser, PermissionScope, SshPublicKey + DeleteRetentionPolicy, RestorePolicyProperties, ChangeFeed, LocalUser, PermissionScope, SshPublicKey, \ + ManagementPolicy, ManagementPolicySchema, ManagementPolicyRule, RuleType, ManagementPolicyDefinition, \ + ManagementPolicyAction, ManagementPolicyFilter, ManagementPolicyBaseBlob, DateAfterModification from azure.mgmt.resource.locks.models import ManagementLockObject from azure.mgmt.dataprotection.models import BackupInstanceResource, BackupInstance, PolicyInfo, Datasource from ..util.azure import azure_client_storage, azure_client_locks, azure_backup_client @@ -204,9 +206,9 @@ def create_or_update_bucket(self, namespace, name, spec): if existing_username not in users_from_spec: self._storage_client.local_users.delete(self._resource_group, bucket_name, existing_username) - if backup_enabled: - storage_account = self._storage_client.storage_accounts.get_properties(self._resource_group, bucket_name) + storage_account = self._storage_client.storage_accounts.get_properties(self._resource_group, bucket_name) + if backup_enabled: vault_name = _backend_config("backup.vault_name", fail_if_missing=True) policy_name = _backend_config("backup.policy_name", fail_if_missing=True) @@ -235,6 +237,15 @@ def create_or_update_bucket(self, namespace, name, spec): parameters=backup_properties ).result() + lifecycle_policy = self._map_lifecycle_policy(spec) + if lifecycle_policy is not None: + self._storage_client.management_policies.create_or_update( + resource_group_name=self._resource_group, + account_name=storage_account.name, + management_policy_name="default", + properties=lifecycle_policy + ) + # Credentials for key in self._storage_client.storage_accounts.list_keys(self._resource_group, bucket_name).keys: if key.key_name == "key1": @@ -246,7 +257,6 @@ def create_or_update_bucket(self, namespace, name, spec): } raise Exception("Could not find keys in azure") - def delete_bucket(self, namespace, name): bucket_name = _calc_name(namespace, name) delete_fake = _backend_config("delete_fake", default=False) @@ -296,6 +306,26 @@ def _map_network_rules(self, spec, public_access): default_action="Allow" if public_access else "Deny" ) + def _map_lifecycle_policy(self, spec): + lifecycle_rules = [] + spec_lifecycle_rules = field_from_spec(spec, "lifecycle.rules", []) + if len(spec_lifecycle_rules) == 0: + return None + for index, rule in enumerate(spec_lifecycle_rules): + lifecycle_rules.append(ManagementPolicyRule(name=f"rule-{index}", + type=RuleType.LIFECYCLE, + definition=ManagementPolicyDefinition( + actions=ManagementPolicyAction( + base_blob=ManagementPolicyBaseBlob( + delete=DateAfterModification( + days_after_modification_greater_than=rule[ + "deleteDaysAfterModification"]))), + filters=ManagementPolicyFilter( + blob_types=["blockBlob"], + prefix_match=[rule["blobPrefix"]])), + enabled=True)) + return ManagementPolicy(policy=ManagementPolicySchema(rules=lifecycle_rules)) + def _get_backup_lock(self, bucket_name): try: return self._lock_client.management_locks.get_at_resource_level( From e3a3199bffbc3a12db4a0b85bb321c0d909e08f5 Mon Sep 17 00:00:00 2001 From: "Valentin J. Hauner" Date: Tue, 10 Jan 2023 14:16:48 +0100 Subject: [PATCH 2/5] Refines README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba6c3e2..c63b62e 100644 --- a/README.md +++ b/README.md @@ -194,8 +194,8 @@ spec: enabled: false # Override the default backup strategy configured in the global operator config lifecycle: # Define rules which determine the lifecycle of blob resources rules: - - blobPrefix: foobar - deleteDaysAfterModification: 30 + - blobPrefix: foobar # Prefix of blob resources to apply rule to, required + deleteDaysAfterModification: 30 # Delete blob resources after number of days after last modification, required containers: # Only relevant for azure, list of containers to create in the bucket, for azure at least one is required, containers not on the list will be removed from the storage account, including their data - name: assets # Name of the container, required anonymousAccess: false # If set to true objects in the container can be accessed without authentication/authorization, only relevant if `security.anonymousAccess` is set to true, optional From 2abde35ebc7f35fe05c2640cc3d06499858061a2 Mon Sep 17 00:00:00 2001 From: "Valentin J. Hauner" Date: Tue, 10 Jan 2023 15:08:12 +0100 Subject: [PATCH 3/5] Adds support for deleting the whole lifecycle policy --- hybridcloud/backends/azureblob.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hybridcloud/backends/azureblob.py b/hybridcloud/backends/azureblob.py index 7381788..eb6c183 100644 --- a/hybridcloud/backends/azureblob.py +++ b/hybridcloud/backends/azureblob.py @@ -245,6 +245,22 @@ def create_or_update_bucket(self, namespace, name, spec): management_policy_name="default", properties=lifecycle_policy ) + else: + try: + # The following call will yield a ResourceNotFoundError iff the policy does not exist, + # hence we will not try to delete it + self._storage_client.management_policies.get( + resource_group_name=self._resource_group, + account_name=storage_account.name, + management_policy_name="default", + ) + self._storage_client.management_policies.delete( + resource_group_name=self._resource_group, + account_name=storage_account.name, + management_policy_name="default", + ) + except ResourceNotFoundError: + pass # Credentials for key in self._storage_client.storage_accounts.list_keys(self._resource_group, bucket_name).keys: From fedbb410999c93060878806471778f973fce84ce Mon Sep 17 00:00:00 2001 From: "Valentin J. Hauner" Date: Tue, 10 Jan 2023 15:20:48 +0100 Subject: [PATCH 4/5] Adds support for defining rule names (optional) --- README.md | 5 +++-- .../templates/objectstorage.yaml | 2 ++ hybridcloud/backends/azureblob.py | 6 +++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c63b62e..96f3ce6 100644 --- a/README.md +++ b/README.md @@ -192,9 +192,10 @@ spec: retentionPeriodInDays: 1 # Days to keep deleted data, optional backup: enabled: false # Override the default backup strategy configured in the global operator config - lifecycle: # Define rules which determine the lifecycle of blob resources + lifecycle: # Define rules which determine the lifecycle of blob resources, optional rules: - - blobPrefix: foobar # Prefix of blob resources to apply rule to, required + - name: foobar-rule # Lifecycle rule name, optional + blobPrefix: foobar # Prefix of blob resources to apply rule to, required deleteDaysAfterModification: 30 # Delete blob resources after number of days after last modification, required containers: # Only relevant for azure, list of containers to create in the bucket, for azure at least one is required, containers not on the list will be removed from the storage account, including their data - name: assets # Name of the container, required diff --git a/helm/hybrid-cloud-object-storage-operator-crds/templates/objectstorage.yaml b/helm/hybrid-cloud-object-storage-operator-crds/templates/objectstorage.yaml index 5463064..e8868bd 100644 --- a/helm/hybrid-cloud-object-storage-operator-crds/templates/objectstorage.yaml +++ b/helm/hybrid-cloud-object-storage-operator-crds/templates/objectstorage.yaml @@ -110,6 +110,8 @@ spec: items: type: object properties: + name: + type: string blobPrefix: type: string deleteDaysAfterModification: diff --git a/hybridcloud/backends/azureblob.py b/hybridcloud/backends/azureblob.py index eb6c183..e50b35e 100644 --- a/hybridcloud/backends/azureblob.py +++ b/hybridcloud/backends/azureblob.py @@ -328,7 +328,11 @@ def _map_lifecycle_policy(self, spec): if len(spec_lifecycle_rules) == 0: return None for index, rule in enumerate(spec_lifecycle_rules): - lifecycle_rules.append(ManagementPolicyRule(name=f"rule-{index}", + if "name" in rule: + rule_name = rule["name"] + else: + rule_name = f"rule-{index}" + lifecycle_rules.append(ManagementPolicyRule(name=rule_name, type=RuleType.LIFECYCLE, definition=ManagementPolicyDefinition( actions=ManagementPolicyAction( From 945c1528ae5effaf72d21c4d60e5a057c1fdc7d5 Mon Sep 17 00:00:00 2001 From: "Valentin J. Hauner" Date: Thu, 12 Jan 2023 12:54:20 +0100 Subject: [PATCH 5/5] Incorporates minor refactoring suggestions from swoehrl-mw --- hybridcloud/backends/azureblob.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/hybridcloud/backends/azureblob.py b/hybridcloud/backends/azureblob.py index e50b35e..80bc926 100644 --- a/hybridcloud/backends/azureblob.py +++ b/hybridcloud/backends/azureblob.py @@ -17,6 +17,7 @@ TAGS_PREFIX = "hybridcloud-object-storage-operator" HTTP_METHODS = ["DELETE", "GET", "HEAD", "MERGE", "OPTIONS", "PATCH", "POST", "PUT"] SFTP_USER_PERMISSIONS = ["READ", "WRITE", "DELETE", "LIST", "CREATE"] +LIFECYCLE_MANAGEMENT_POLICY_NAME = "default" def _backend_config(key, default=None, fail_if_missing=False): @@ -242,7 +243,7 @@ def create_or_update_bucket(self, namespace, name, spec): self._storage_client.management_policies.create_or_update( resource_group_name=self._resource_group, account_name=storage_account.name, - management_policy_name="default", + management_policy_name=LIFECYCLE_MANAGEMENT_POLICY_NAME, properties=lifecycle_policy ) else: @@ -252,12 +253,12 @@ def create_or_update_bucket(self, namespace, name, spec): self._storage_client.management_policies.get( resource_group_name=self._resource_group, account_name=storage_account.name, - management_policy_name="default", + management_policy_name=LIFECYCLE_MANAGEMENT_POLICY_NAME, ) self._storage_client.management_policies.delete( resource_group_name=self._resource_group, account_name=storage_account.name, - management_policy_name="default", + management_policy_name=LIFECYCLE_MANAGEMENT_POLICY_NAME, ) except ResourceNotFoundError: pass @@ -328,10 +329,7 @@ def _map_lifecycle_policy(self, spec): if len(spec_lifecycle_rules) == 0: return None for index, rule in enumerate(spec_lifecycle_rules): - if "name" in rule: - rule_name = rule["name"] - else: - rule_name = f"rule-{index}" + rule_name = rule.get("name", f"rule-{index}") lifecycle_rules.append(ManagementPolicyRule(name=rule_name, type=RuleType.LIFECYCLE, definition=ManagementPolicyDefinition(