From 4db98a9cfc0ca9a024b4f9742670fa1550210925 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 12 Sep 2023 12:47:55 -0700 Subject: [PATCH 01/13] Update support for multipart uploads to non-AWS S3 endpoints --- fence/blueprints/data/indexd.py | 11 ++++++----- fence/blueprints/data/multipart_upload.py | 12 ++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index f2978f56d..ca88d8c64 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -349,12 +349,13 @@ def init_multipart_upload(key, expires_in=None, bucket=None): Returns: uploadId(str) """ - bucket = bucket or flask.current_app.config["DATA_UPLOAD_BUCKET"] if not bucket: - raise InternalError( - "fence not configured with data upload bucket; can't create signed URL" - ) - + try: + bucket = flask.current_app.config["DATA_UPLOAD_BUCKET"] + except KeyError: + raise InternalError( + "fence not configured with data upload bucket; can't create signed URL" + ) s3_url = "s3://{}/{}".format(bucket, key) return S3IndexedFileLocation(s3_url).init_multipart_upload(expires_in) diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index 4adc7f5b3..7352f66f1 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -39,7 +39,6 @@ def initialize_multipart_upload(bucket_name, key, credentials): aws_secret_access_key=credentials["aws_secret_access_key"], aws_session_token=credentials.get("aws_session_token"), ) - s3client = None if url: s3client = session.client("s3", endpoint_url=url) @@ -94,7 +93,6 @@ def complete_multipart_upload(bucket_name, key, credentials, uploadId, parts): aws_secret_access_key=credentials["aws_secret_access_key"], aws_session_token=credentials.get("aws_session_token"), ) - s3client = None if url: s3client = session.client("s3", endpoint_url=url) @@ -147,7 +145,12 @@ def generate_presigned_url_for_uploading_part( ) bucket = s3_buckets.get(bucket_name) - if s3_buckets.get("endpoint_url"): + s3_buckets = get_value( + config, "S3_BUCKETS", InternalError("S3_BUCKETS not configured") + ) + bucket = s3_buckets.get(bucket_name) + + if bucket.get("endpoint_url"): url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( bucket_name, key.strip("/") ) @@ -156,9 +159,10 @@ def generate_presigned_url_for_uploading_part( additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId} try: - return generate_aws_presigned_url( + presigned_url = generate_aws_presigned_url( url, "PUT", credentials, "s3", region, expires, additional_signed_qs ) + return presigned_url except Exception as e: raise InternalError( "Can not generate presigned url for part number {} of key {}. Detail {}".format( From bb0883a5eb487be21b2aee5a555b9625318fa2ca Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 7 Dec 2023 10:23:56 -0800 Subject: [PATCH 02/13] Add initial endpoint_url parameter to boto3 client --- fence/resources/aws/boto_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index 93a5366ed..f9ac792f2 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -19,7 +19,7 @@ class BotoManager(object): def __init__(self, config, logger): self.sts_client = client("sts", **config) - self.s3_client = client("s3", **config) + self.s3_client = client("s3", endpoint_url='TODO', **config) self.logger = logger self.ec2 = None self.iam = None From 6703a918b202c26304b13b1ef209d6b9aad92204 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 11 Dec 2023 10:48:29 -0800 Subject: [PATCH 03/13] Add initial multi-client support to Boto Manager --- fence/__init__.py | 3 ++- fence/blueprints/data/indexd.py | 1 + fence/resources/aws/boto_manager.py | 27 +++++++++++++++++++++++---- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/fence/__init__.py b/fence/__init__.py index 2d44e33f2..1fcc4b24e 100755 --- a/fence/__init__.py +++ b/fence/__init__.py @@ -389,7 +389,8 @@ def app_config( def _setup_data_endpoint_and_boto(app): if "AWS_CREDENTIALS" in config and len(config["AWS_CREDENTIALS"]) > 0: value = list(config["AWS_CREDENTIALS"].values())[0] - app.boto = BotoManager(value, logger=logger) + buckets = config.get("S3_BUCKETS", {}) + app.boto = BotoManager(value, buckets, logger=logger) app.register_blueprint(fence.blueprints.data.blueprint, url_prefix="/data") diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index ca88d8c64..96cb4d21c 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1150,6 +1150,7 @@ def complete_multipart_upload(self, uploadId, parts, expires_in): def delete(self, bucket, file_id): try: + print(f"DEBUG boto: {flask.current_app.boto}") return flask.current_app.boto.delete_data_file(bucket, file_id) except Exception as e: logger.error(e) diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index f9ac792f2..ebac01071 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -17,13 +17,31 @@ class BotoManager(object): 900 # minimum time for aws assume role is 900 seconds as per boto docs ) - def __init__(self, config, logger): + def __init__(self, config, buckets, logger): self.sts_client = client("sts", **config) - self.s3_client = client("s3", endpoint_url='TODO', **config) + self.s3_client = client("s3", **config) + self.s3_clients = self.create_s3_clients(config, buckets) self.logger = logger self.ec2 = None self.iam = None + def create_s3_clients(self, config, buckets): + s3_clients = { + 'default': client('s3', **config) + } + for bucket in buckets: + print(f"DEBUG bucket: {bucket}") + if buckets[bucket]['endpoint_url'] is not None: + print(f"DEBUG endpoint_url: {endpoint_url}") + endpoint_url = buckets[bucket]['endpoint_url'] + s3_clients[bucket] = client('s3', **config, endpoint_url=endpoint_url) + return s3_clients + + def get_s3_client(self, bucket): + if self.s3_clients.get(bucket) is None: + return self.s3_clients['default'] + return self.s3_clients[bucket] + def delete_data_file(self, bucket, prefix): """ We use buckets with versioning disabled. @@ -33,7 +51,8 @@ def delete_data_file(self, bucket, prefix): https://docs.aws.amazon.com/AmazonS3/latest/dev/DeletingObjectsfromVersioningSuspendedBuckets.html """ try: - s3_objects = self.s3_client.list_objects_v2( + s3_client = self.get_s3_client(bucket) + s3_objects = s3_client.list_objects_v2( Bucket=bucket, Prefix=prefix, Delimiter="/" ) @@ -52,7 +71,7 @@ def delete_data_file(self, bucket, prefix): self.logger.error("multiple files found with prefix {}".format(prefix)) return ("Multiple files found matching this prefix. Backing off.", 400) key = s3_objects["Contents"][0]["Key"] - self.s3_client.delete_object(Bucket=bucket, Key=key) + s3_client.delete_object(Bucket=bucket, Key=key) self.logger.info( "deleted file for prefix {} in bucket {}".format(prefix, bucket) ) From 3839a4d625ebbd64a082856b1e994f95687e3019 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 15 Dec 2023 11:22:41 -0800 Subject: [PATCH 04/13] Remove debugging statements --- fence/blueprints/data/indexd.py | 1 - fence/blueprints/data/multipart_upload.py | 3 +-- fence/resources/aws/boto_manager.py | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 96cb4d21c..ca88d8c64 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -1150,7 +1150,6 @@ def complete_multipart_upload(self, uploadId, parts, expires_in): def delete(self, bucket, file_id): try: - print(f"DEBUG boto: {flask.current_app.boto}") return flask.current_app.boto.delete_data_file(bucket, file_id) except Exception as e: logger.error(e) diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index 7352f66f1..948c66a5b 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -159,10 +159,9 @@ def generate_presigned_url_for_uploading_part( additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId} try: - presigned_url = generate_aws_presigned_url( + return generate_aws_presigned_url( url, "PUT", credentials, "s3", region, expires, additional_signed_qs ) - return presigned_url except Exception as e: raise InternalError( "Can not generate presigned url for part number {} of key {}. Detail {}".format( diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index ebac01071..3cb4c68dd 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -30,9 +30,8 @@ def create_s3_clients(self, config, buckets): 'default': client('s3', **config) } for bucket in buckets: - print(f"DEBUG bucket: {bucket}") - if buckets[bucket]['endpoint_url'] is not None: - print(f"DEBUG endpoint_url: {endpoint_url}") + if 'endpoint_url' in buckets[bucket]: + endpoint_url = buckets[bucket]['endpoint_url'] endpoint_url = buckets[bucket]['endpoint_url'] s3_clients[bucket] = client('s3', **config, endpoint_url=endpoint_url) return s3_clients From fbcb64cc69e2f362dbe67a9dca9bfe1cc72f5c29 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 15 Dec 2023 11:26:14 -0800 Subject: [PATCH 05/13] Revert changes to indexd data blueprint --- fence/blueprints/data/indexd.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index ca88d8c64..f2978f56d 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -349,13 +349,12 @@ def init_multipart_upload(key, expires_in=None, bucket=None): Returns: uploadId(str) """ + bucket = bucket or flask.current_app.config["DATA_UPLOAD_BUCKET"] if not bucket: - try: - bucket = flask.current_app.config["DATA_UPLOAD_BUCKET"] - except KeyError: - raise InternalError( - "fence not configured with data upload bucket; can't create signed URL" - ) + raise InternalError( + "fence not configured with data upload bucket; can't create signed URL" + ) + s3_url = "s3://{}/{}".format(bucket, key) return S3IndexedFileLocation(s3_url).init_multipart_upload(expires_in) From 36e86cd7e5a4d6792b2a99dc8a8f04b8d3edd20a Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 15 Dec 2023 11:30:50 -0800 Subject: [PATCH 06/13] Revert changes to multipart_upload.py --- fence/blueprints/data/multipart_upload.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fence/blueprints/data/multipart_upload.py b/fence/blueprints/data/multipart_upload.py index 948c66a5b..4adc7f5b3 100644 --- a/fence/blueprints/data/multipart_upload.py +++ b/fence/blueprints/data/multipart_upload.py @@ -39,6 +39,7 @@ def initialize_multipart_upload(bucket_name, key, credentials): aws_secret_access_key=credentials["aws_secret_access_key"], aws_session_token=credentials.get("aws_session_token"), ) + s3client = None if url: s3client = session.client("s3", endpoint_url=url) @@ -93,6 +94,7 @@ def complete_multipart_upload(bucket_name, key, credentials, uploadId, parts): aws_secret_access_key=credentials["aws_secret_access_key"], aws_session_token=credentials.get("aws_session_token"), ) + s3client = None if url: s3client = session.client("s3", endpoint_url=url) @@ -145,12 +147,7 @@ def generate_presigned_url_for_uploading_part( ) bucket = s3_buckets.get(bucket_name) - s3_buckets = get_value( - config, "S3_BUCKETS", InternalError("S3_BUCKETS not configured") - ) - bucket = s3_buckets.get(bucket_name) - - if bucket.get("endpoint_url"): + if s3_buckets.get("endpoint_url"): url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( bucket_name, key.strip("/") ) From 83032c4bcd4242dbbaac7eb3967405ef5b2761c0 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Fri, 15 Dec 2023 11:34:47 -0800 Subject: [PATCH 07/13] Revert change to s3_client call in Boto Manager --- fence/resources/aws/boto_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index 3cb4c68dd..3b60b7ca8 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -70,7 +70,7 @@ def delete_data_file(self, bucket, prefix): self.logger.error("multiple files found with prefix {}".format(prefix)) return ("Multiple files found matching this prefix. Backing off.", 400) key = s3_objects["Contents"][0]["Key"] - s3_client.delete_object(Bucket=bucket, Key=key) + self.s3_client.delete_object(Bucket=bucket, Key=key) self.logger.info( "deleted file for prefix {} in bucket {}".format(prefix, bucket) ) From 915edd22dd52451555f980c7644404fda05e6aae Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 26 Dec 2023 17:13:56 -0800 Subject: [PATCH 08/13] Update BotoManager to use specified credentials for each bucket --- fence/__init__.py | 4 ++-- fence/resources/aws/boto_manager.py | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/fence/__init__.py b/fence/__init__.py index 1fcc4b24e..ea58cf02c 100755 --- a/fence/__init__.py +++ b/fence/__init__.py @@ -388,9 +388,9 @@ def app_config( def _setup_data_endpoint_and_boto(app): if "AWS_CREDENTIALS" in config and len(config["AWS_CREDENTIALS"]) > 0: - value = list(config["AWS_CREDENTIALS"].values())[0] + creds = config["AWS_CREDENTIALS"] buckets = config.get("S3_BUCKETS", {}) - app.boto = BotoManager(value, buckets, logger=logger) + app.boto = BotoManager(creds, buckets, logger=logger) app.register_blueprint(fence.blueprints.data.blueprint, url_prefix="/data") diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index 3b60b7ca8..1fee8e8ca 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -26,14 +26,22 @@ def __init__(self, config, buckets, logger): self.iam = None def create_s3_clients(self, config, buckets): - s3_clients = { - 'default': client('s3', **config) - } + s3_clients = {} + for creds in config: + if config[creds]['aws'] == 'true': + s3_clients = {'default': self.s3_client} + for bucket in buckets: + cred_name = buckets[bucket]['cred'] + keys = config[cred_name] + if 'aws' in keys: + keys.pop('aws') + if 'endpoint_url' in buckets[bucket]: endpoint_url = buckets[bucket]['endpoint_url'] - endpoint_url = buckets[bucket]['endpoint_url'] - s3_clients[bucket] = client('s3', **config, endpoint_url=endpoint_url) + s3_clients[bucket] = client('s3', **keys, endpoint_url=endpoint_url) + else: + s3_clients[bucket] = client('s3', **keys) return s3_clients def get_s3_client(self, bucket): From 7893d17647169c20c8ed01ae62be6fedeb09e438 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 22 Jan 2024 13:05:14 -0800 Subject: [PATCH 09/13] Resolve unit tests --- fence/blueprints/data/indexd.py | 5 +++-- fence/resources/aws/boto_manager.py | 20 +++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index f2978f56d..09c81f26e 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -349,8 +349,9 @@ def init_multipart_upload(key, expires_in=None, bucket=None): Returns: uploadId(str) """ - bucket = bucket or flask.current_app.config["DATA_UPLOAD_BUCKET"] - if not bucket: + try: + bucket = bucket or flask.current_app.config["DATA_UPLOAD_BUCKET"] + except KeyError: raise InternalError( "fence not configured with data upload bucket; can't create signed URL" ) diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index 1fee8e8ca..215040e9c 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -18,30 +18,28 @@ class BotoManager(object): ) def __init__(self, config, buckets, logger): - self.sts_client = client("sts", **config) - self.s3_client = client("s3", **config) + default = list(config.values())[0] + self.sts_client = client("sts", **default) + self.s3_client = client("s3", **default) self.s3_clients = self.create_s3_clients(config, buckets) self.logger = logger self.ec2 = None self.iam = None def create_s3_clients(self, config, buckets): - s3_clients = {} - for creds in config: - if config[creds]['aws'] == 'true': - s3_clients = {'default': self.s3_client} + s3_clients = {'default': self.s3_client} for bucket in buckets: cred_name = buckets[bucket]['cred'] - keys = config[cred_name] - if 'aws' in keys: - keys.pop('aws') + creds = {} + if cred_name != '*': + creds = config[cred_name] if 'endpoint_url' in buckets[bucket]: endpoint_url = buckets[bucket]['endpoint_url'] - s3_clients[bucket] = client('s3', **keys, endpoint_url=endpoint_url) + s3_clients[bucket] = client('s3', **creds, endpoint_url=endpoint_url) else: - s3_clients[bucket] = client('s3', **keys) + s3_clients[bucket] = client('s3', **creds) return s3_clients def get_s3_client(self, bucket): From f64baffac1a7a5d76ccb7e836a0b7fcd835f7670 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 25 Jan 2024 11:41:02 -0800 Subject: [PATCH 10/13] Use correct s3_client to delete files --- fence/resources/aws/boto_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index 215040e9c..7493596e9 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -76,7 +76,7 @@ def delete_data_file(self, bucket, prefix): self.logger.error("multiple files found with prefix {}".format(prefix)) return ("Multiple files found matching this prefix. Backing off.", 400) key = s3_objects["Contents"][0]["Key"] - self.s3_client.delete_object(Bucket=bucket, Key=key) + s3_client.delete_object(Bucket=bucket, Key=key) self.logger.info( "deleted file for prefix {} in bucket {}".format(prefix, bucket) ) From 51f83fd97426f997f4dd2655ede3229ea8bd9fe5 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 29 Feb 2024 12:45:06 -0800 Subject: [PATCH 11/13] Add unit tests for creating S3 clients and deleting files --- tests/data/test_boto_manager.py | 106 ++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/data/test_boto_manager.py diff --git a/tests/data/test_boto_manager.py b/tests/data/test_boto_manager.py new file mode 100644 index 000000000..5079d7820 --- /dev/null +++ b/tests/data/test_boto_manager.py @@ -0,0 +1,106 @@ +import pytest +from unittest.mock import MagicMock, patch +from fence.resources.aws.boto_manager import BotoManager + + +class TestData: + """Generate bucket test data that aims to mirror the default example Fence config file.""" + def __init__(self): + self.config = {} + self.buckets = {} + + def single_bucket(self): + self.config = { + 'CRED1': {'access_key': 'key1', 'secret_key': 'secret1'}, + } + self.buckets = { + 'bucket1': {'cred': 'CRED1', 'region': 'us-east-1', 'endpoint_url': 'https://example.com'}, + } + return self + + def multiple_buckets(self): + single_bucket = self.single_bucket() + self.config = single_bucket.config | { + 'CRED2': {'access_key': 'key2', 'secret_key': 'secret2'}, + } + self.buckets = single_bucket.buckets | { + 'bucket2': {'cred': 'CRED2', 'region': 'us-east-1'}, + 'bucket3': {'cred': '*'}, + 'bucket4': {'cred': 'CRED1', 'region': 'us-east-1', 'role-arn': 'arn:aws:iam::role1'} + } + return self + + +@patch('fence.resources.aws.boto_manager.client') +def test_create_s3_client_single(mock_client): + test_data = TestData().single_bucket() + config = test_data.config + buckets = test_data.buckets + logger = MagicMock() + boto_manager = BotoManager(config, buckets, logger) + + s3_clients = boto_manager.create_s3_clients(config, buckets) + + # Assert that the correct call was made to the client function + mock_client.assert_any_call('s3', access_key='key1', secret_key='key1', endpoint_url='https://example.com') + + # Assert that the returned dictionary contains the correct client + assert len(s3_clients) == 1 + assert 'bucket1' in s3_clients + + +@patch('fence.resources.aws.boto_manager.client') +def test_create_s3_clients_multiple(mock_client): + test_data = TestData().multiple_buckets() + config = test_data.config + buckets = test_data.buckets + logger = MagicMock() + boto_manager = BotoManager(config, buckets, logger) + + # Call the method under test + s3_clients = boto_manager.create_s3_clients(config, buckets) + + # Assert that the correct calls were made to the client function + mock_client.assert_any_call('s3', access_key='key1', secret_key='secret1', endpoint_url='https://example.com') + mock_client.assert_any_call('s3', access_key='key2', secret_key='secret2') + mock_client.assert_any_call('s3') + mock_client.assert_any_call('s3', access_key='key1', secret_key='secret1') + + # Assert that the returned dictionary contains the correct clients + assert len(s3_clients) == 4 + assert 'bucket1' in s3_clients + assert 'bucket2' in s3_clients + assert 'bucket3' in s3_clients + assert 'bucket4' in s3_clients + + +@pytest.mark.parametrize("bucket", ['bucket1', 'bucket2', 'bucket3', 'bucket4']) +@patch('fence.resources.aws.boto_manager.client') +def test_delete_data_file(mock_client, bucket): + test_data = TestData().multiple_buckets() + config = test_data.config + buckets = test_data.buckets + logger = MagicMock() + boto_manager = BotoManager(config, buckets, logger) + + # Mock the response of list_objects_v2 to include the desired key + prefix = 'data/file.txt' + mock_list_objects_v2_response = { + 'Contents': [{'Key': prefix}] + } + # Set up the mock S3 client and its list_objects_v2 and delete_object methods + mock_s3_client = mock_client.return_value + mock_s3_client.list_objects_v2.return_value = mock_list_objects_v2_response + + result = boto_manager.delete_data_file(bucket, prefix) + + # Create S3 clients for each of the buckets + _ = boto_manager.create_s3_clients(config, buckets) + s3_client = boto_manager.get_s3_client(bucket) + s3_client.list_objects_v2.assert_called_once_with( + Bucket=bucket, Prefix=prefix, Delimiter="/" + ) + s3_client.delete_object.assert_called_once_with(Bucket=bucket, Key='data/file.txt') + + # Assert the expected result + assert result == ("", 204) From d7f91d50411f53e340f82abf26ab4b6c27ad00e0 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 29 Feb 2024 12:59:39 -0800 Subject: [PATCH 12/13] Update Boto Manager --- fence/resources/aws/boto_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fence/resources/aws/boto_manager.py b/fence/resources/aws/boto_manager.py index 7493596e9..e0744554a 100644 --- a/fence/resources/aws/boto_manager.py +++ b/fence/resources/aws/boto_manager.py @@ -27,14 +27,12 @@ def __init__(self, config, buckets, logger): self.iam = None def create_s3_clients(self, config, buckets): - s3_clients = {'default': self.s3_client} - + s3_clients = {} for bucket in buckets: cred_name = buckets[bucket]['cred'] creds = {} if cred_name != '*': creds = config[cred_name] - if 'endpoint_url' in buckets[bucket]: endpoint_url = buckets[bucket]['endpoint_url'] s3_clients[bucket] = client('s3', **creds, endpoint_url=endpoint_url) @@ -44,7 +42,7 @@ def create_s3_clients(self, config, buckets): def get_s3_client(self, bucket): if self.s3_clients.get(bucket) is None: - return self.s3_clients['default'] + return self.s3_clients[0] return self.s3_clients[bucket] def delete_data_file(self, bucket, prefix): From acfe08f10dd2f32425a75fa03be6010feb2c46ab Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Thu, 29 Feb 2024 14:24:31 -0800 Subject: [PATCH 13/13] Add minor fix to resolve S3 Mock test --- tests/data/test_boto_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/test_boto_manager.py b/tests/data/test_boto_manager.py index 5079d7820..2db6a8c12 100644 --- a/tests/data/test_boto_manager.py +++ b/tests/data/test_boto_manager.py @@ -42,7 +42,7 @@ def test_create_s3_client_single(mock_client): s3_clients = boto_manager.create_s3_clients(config, buckets) # Assert that the correct call was made to the client function - mock_client.assert_any_call('s3', access_key='key1', secret_key='key1', endpoint_url='https://example.com') + mock_client.assert_any_call('s3', access_key='key1', secret_key='secret1', endpoint_url='https://example.com') # Assert that the returned dictionary contains the correct client assert len(s3_clients) == 1