Change versioned State S3 Buckets deletion to use a Python script

finos · Feb 6, 2024 · b4da2e8 · b4da2e8
1 parent 4dc021b
commit b4da2e8
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 11 deletions.
diff --git a/deployment/init_grid/cloudformation/Makefile b/deployment/init_grid/cloudformation/Makefile
@@ -23,7 +23,7 @@ $(BUILD_DIR)/grid_init/tag.$(TAG):
 	@touch $(BUILD_DIR)/grid_init/tag.$(TAG)
 
 $(BUILD_DIR)/grid_init/init-grid-$(TAG): $(BUILD_DIR)/grid_init/tag.$(TAG) $(YAML_SRC)
-	@if [ "$(shell aws cloudformation list-stacks --query 'StackSummaries[?StackStatus==`CREATE_COMPLETE` && StackName==`$(TAG)`].StackName' --output text)" != "$(TAG)" ]; then \
+	@if [ "$(shell aws cloudformation list-stacks --region $(REGION) --query 'StackSummaries[?StackStatus==`CREATE_COMPLETE` && StackName==`$(TAG)`].StackName' --output text)" != "$(TAG)" ]; then \
 	echo "CloudFormation stack $(TAG) doesn't exist. Creating ..."; \
 	aws cloudformation create-stack --stack-name $(TAG) --region $(REGION) --template-body file://$(YAML_SRC) --parameters ParameterKey=BucketTag,ParameterValue=$(TAG); else \
 	echo "Failed creating the CloudFormation stack as it may already exist. Trying to update ..."; \
@@ -38,19 +38,13 @@ $(TAG)-workload-bucket: $(BUILD_DIR)/grid_init/init-grid-$(TAG)-workload-bucket
 init: $(BUILD_DIR)/grid_init
 
 clean-grid-state:
-	aws --region $(REGION) s3 rm --recursive s3://$(GRID_TFSTATE_BUCKET) || true
-	@aws s3api delete-objects --bucket $(GRID_TFSTATE_BUCKET) --delete '$(shell aws s3api list-object-versions --bucket $(GRID_TFSTATE_BUCKET) --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')' --output text > /dev/null 2>&1 || true
-	@aws s3api delete-objects --bucket $(GRID_TFSTATE_BUCKET) --delete '$(shell aws s3api list-object-versions --bucket $(GRID_TFSTATE_BUCKET) --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}')' --output text > /dev/null 2>&1 || true
+	../scripts/python/delete_bucket.py $(GRID_TFSTATE_BUCKET)
 
 clean-image-state:
-	aws --region $(REGION) s3 rm --recursive s3://$(IMAGE_TFSTATE_BUCKET) || true
-	@aws s3api delete-objects --bucket $(IMAGE_TFSTATE_BUCKET) --delete '$(shell aws s3api list-object-versions --bucket $(IMAGE_TFSTATE_BUCKET) --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')' --output text > /dev/null 2>&1 || true
-	@aws s3api delete-objects --bucket $(IMAGE_TFSTATE_BUCKET) --delete '$(shell aws s3api list-object-versions --bucket $(IMAGE_TFSTATE_BUCKET) --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}')' --output text > /dev/null 2>&1 || true
+	../scripts/python/delete_bucket.py $(IMAGE_TFSTATE_BUCKET)
 
 clean-lambda-storage:
-	aws --region $(REGION) s3 rm --recursive s3://$(LAMBDA_LAYER_BUCKET) || true
-	@aws s3api delete-objects --bucket $(LAMBDA_LAYER_BUCKET) --delete '$(shell aws s3api list-object-versions --bucket $(LAMBDA_LAYER_BUCKET) --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')' > /dev/null 2>&1 || true
-	@aws s3api delete-objects --bucket $(LAMBDA_LAYER_BUCKET) --delete '$(shell aws s3api list-object-versions --bucket $(LAMBDA_LAYER_BUCKET) --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}')' --output text > /dev/null 2>&1 || true
+	../scripts/python/delete_bucket.py $(LAMBDA_LAYER_BUCKET)
 
 clean: clean-image-state clean-grid-state clean-lambda-storage
 	rm -rf $(BUILD_DIR)/grid_init/tag.*

diff --git a/deployment/init_grid/scripts/python/delete_bucket.py b/deployment/init_grid/scripts/python/delete_bucket.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+import sys
+import boto3
+
+if len(sys.argv) < 2:
+    print("Please provide one bucket name as a command line argument")
+    sys.exit(1)
+
+bucket_name = sys.argv[1]
+
+s3 = boto3.resource("s3")
+bucket = s3.Bucket(bucket_name)
+
+s3_client = boto3.client("s3")
+versioning = s3_client.get_bucket_versioning(Bucket=bucket_name)
+
+if versioning.get("Status") == "Enabled":
+    bucket.object_versions.delete()
+    print(f"Succesfully deleted all bucket object versions in bucket: {bucket_name}")
+else:
+    bucket.objects.delete()
+    print(f"Succesfully deleted all bucket objects in bucket: {bucket_name}")
+
+
+s3_client.delete_bucket(Bucket=bucket_name)
+print(f"Succesfully deleted bucket: {bucket_name}")
diff --git a/examples/configurations/Makefile b/examples/configurations/Makefile
@@ -10,7 +10,7 @@ FILE_HANDLER=
 FUNCTION_HANDLER=
 ACCOUNT_ID?=$(shell aws sts get-caller-identity | jq -r '.Account')
 BUCKET_NAME=$(shell aws cloudformation describe-stacks --stack-name $(TAG) --region $(REGION) --query 'Stacks[0].Outputs[?OutputKey==`LambdaLayerBucketID`].OutputValue' --output text)
-S3_KMS_KEY_ARN=$(shell aws kms describe-key --key-id $(shell aws cloudformation describe-stacks --stack-name $(TAG) --region $(REGION) --query 'Stacks[0].Outputs[?OutputKey==`HTCStateS3KeyAlias`].OutputValue' --output text) --query 'KeyMetadata.Arn' --output text)
+S3_KMS_KEY_ARN=$(shell aws kms describe-key --key-id $(shell aws cloudformation describe-stacks --stack-name $(TAG) --region $(REGION) --query 'Stacks[0].Outputs[?OutputKey==`HTCStateS3KeyAlias`].OutputValue' --output text) --region $(REGION) --query 'KeyMetadata.Arn' --output text)
 
 
 generated-c++: grid_config.json.tpl