diff --git a/tests/s3_s3express_client_test.c b/tests/s3_s3express_client_test.c index cad5b81d2..b8f771a7f 100644 --- a/tests/s3_s3express_client_test.c +++ b/tests/s3_s3express_client_test.c @@ -228,8 +228,6 @@ static int s_s3express_client_put_test_helper(struct aws_allocator *allocator, s struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1"); - char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com"; - struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint); struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-test"); struct aws_s3_client_config client_config = { @@ -243,13 +241,23 @@ static int s_s3express_client_put_test_helper(struct aws_allocator *allocator, s struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); ASSERT_SUCCESS(s_s3express_put_object_request( - allocator, client, content_length, &tester, host_cursor, key_cursor, region_cursor)); + allocator, + client, + content_length, + &tester, + g_test_s3express_bucket_use1_az4_endpoint, + key_cursor, + region_cursor)); struct aws_byte_cursor west2_region_cursor = aws_byte_cursor_from_c_str("us-west-2"); - char west2_endpoint[] = "crts-west2--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com"; - struct aws_byte_cursor west2_host_cursor = aws_byte_cursor_from_c_str(west2_endpoint); ASSERT_SUCCESS(s_s3express_put_object_request( - allocator, client, content_length, &tester, west2_host_cursor, key_cursor, west2_region_cursor)); + allocator, + client, + content_length, + &tester, + g_test_s3express_bucket_usw2_az1_endpoint, + key_cursor, + west2_region_cursor)); aws_s3_client_release(client); aws_s3_tester_clean_up(&tester); @@ -280,13 +288,9 @@ TEST_CASE(s3express_client_put_object_multipart_multiple) { struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1"); - char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com"; - struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint); struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-test"); struct aws_byte_cursor west2_region_cursor = aws_byte_cursor_from_c_str("us-west-2"); - char west2_endpoint[] = "crts-west2--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com"; - struct aws_byte_cursor west2_host_cursor = aws_byte_cursor_from_c_str(west2_endpoint); struct aws_s3_client_config client_config = { .part_size = MB_TO_BYTES(5), @@ -302,11 +306,11 @@ TEST_CASE(s3express_client_put_object_multipart_multiple) { input_streams[i] = aws_s3_test_input_stream_new(allocator, MB_TO_BYTES(10)); struct aws_byte_cursor request_region = region_cursor; - struct aws_byte_cursor request_host = host_cursor; + struct aws_byte_cursor request_host = g_test_s3express_bucket_use1_az4_endpoint; if (i % 2 == 0) { /* Make half of request to east1 and rest half to west2 */ request_region = west2_region_cursor; - request_host = west2_host_cursor; + request_host = g_test_s3express_bucket_usw2_az1_endpoint; } struct aws_http_message *message = aws_s3_test_put_object_request_new( @@ -401,8 +405,6 @@ TEST_CASE(s3express_client_put_object_long_running_session_refresh) { struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1"); - char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com"; - struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint); struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-test"); struct aws_s3_client_config client_config = { @@ -421,7 +423,12 @@ TEST_CASE(s3express_client_put_object_long_running_session_refresh) { struct aws_input_stream *upload_stream = aws_s3_test_input_stream_new(allocator, MB_TO_BYTES(10)); struct aws_http_message *message = aws_s3_test_put_object_request_new( - allocator, &host_cursor, key_cursor, g_test_body_content_type, upload_stream, 0); + allocator, + &g_test_s3express_bucket_use1_az4_endpoint, + key_cursor, + g_test_body_content_type, + upload_stream, + 0); struct aws_s3_meta_request_options options; AWS_ZERO_STRUCT(options); options.type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT; @@ -473,10 +480,6 @@ TEST_CASE(s3express_client_get_object) { struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1"); - char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com"; - struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint); - struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-download-10MB"); - struct aws_s3_client_config client_config = { .part_size = MB_TO_BYTES(5), .enable_s3express = true, @@ -487,7 +490,8 @@ TEST_CASE(s3express_client_get_object) { struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); - struct aws_http_message *message = aws_s3_test_get_object_request_new(allocator, host_cursor, key_cursor); + struct aws_http_message *message = aws_s3_test_get_object_request_new( + allocator, g_test_s3express_bucket_use1_az4_endpoint, g_pre_existing_object_10MB); struct aws_s3_meta_request_options options; AWS_ZERO_STRUCT(options); @@ -532,10 +536,6 @@ TEST_CASE(s3express_client_get_object_multiple) { struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1"); - char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com"; - struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint); - struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-download-10MB"); - struct aws_s3_client_config client_config = { .part_size = MB_TO_BYTES(5), .enable_s3express = true, @@ -548,7 +548,8 @@ TEST_CASE(s3express_client_get_object_multiple) { for (size_t i = 0; i < num_meta_requests; ++i) { - struct aws_http_message *message = aws_s3_test_get_object_request_new(allocator, host_cursor, key_cursor); + struct aws_http_message *message = aws_s3_test_get_object_request_new( + allocator, g_test_s3express_bucket_use1_az4_endpoint, g_pre_existing_object_10MB); struct aws_s3_meta_request_options options; AWS_ZERO_STRUCT(options); diff --git a/tests/test_helper/test_helper.py b/tests/test_helper/test_helper.py index 003acbbda..9056dd97a 100755 --- a/tests/test_helper/test_helper.py +++ b/tests/test_helper/test_helper.py @@ -11,9 +11,11 @@ print(boto3.__version__) -REGION = 'us-east-1' +REGION = 'us-west-2' +REGION_EAST_1 = 'us-east-1' s3 = boto3.resource('s3') s3_client = boto3.client('s3', region_name=REGION) +s3_client_east1 = boto3.client('s3', region_name=REGION_EAST_1) s3_control_client = boto3.client('s3control') @@ -50,9 +52,9 @@ def create_bytes(size): return bytearray([1] * size) -def put_pre_existing_objects(size, keyname, bucket=BUCKET_NAME_BASE, sse=None, public_read=False): +def put_pre_existing_objects(size, keyname, bucket=BUCKET_NAME_BASE, sse=None, public_read=False, client=s3_client): if size == 0: - s3_client.put_object(Bucket=bucket, Key=keyname) + client.put_object(Bucket=bucket, Key=keyname) print(f"Object {keyname} uploaded") return @@ -71,7 +73,7 @@ def put_pre_existing_objects(size, keyname, bucket=BUCKET_NAME_BASE, sse=None, p if public_read: args['ACL'] = 'public-read' try: - s3_client.put_object(**args) + client.put_object(**args) except botocore.exceptions.ClientError as e: print(f"Object {keyname} failed to upload, with exception: {e}") if public_read and e.response['Error']['Code'] == 'AccessDenied': @@ -80,7 +82,7 @@ def put_pre_existing_objects(size, keyname, bucket=BUCKET_NAME_BASE, sse=None, p print(f"Object {keyname} uploaded") -def create_bucket_with_lifecycle(availability_zone=None): +def create_bucket_with_lifecycle(availability_zone=None, client=s3_client): try: # Create the bucket. This returns an error if the bucket already exists. @@ -100,13 +102,10 @@ def create_bucket_with_lifecycle(availability_zone=None): bucket_config = {'LocationConstraint': REGION} bucket_name = BUCKET_NAME_BASE - print(bucket_name) - print(bucket_config) - - s3_client.create_bucket( + client.create_bucket( Bucket=bucket_name, CreateBucketConfiguration=bucket_config) if availability_zone is None: - s3_client.put_bucket_lifecycle_configuration( + client.put_bucket_lifecycle_configuration( Bucket=bucket_name, LifecycleConfiguration={ 'Rules': [ @@ -132,7 +131,7 @@ def create_bucket_with_lifecycle(availability_zone=None): print(f"Bucket {bucket_name} created", file=sys.stderr) put_pre_existing_objects( - 10*MB, 'pre-existing-10MB', bucket=bucket_name) + 10*MB, 'pre-existing-10MB', bucket=bucket_name, client=client) if availability_zone is None: put_pre_existing_objects( @@ -190,36 +189,43 @@ def create_bucket_with_public_object(): raise e -def cleanup(bucket_name): - bucket = s3.Bucket(bucket_name) - bucket.objects.all().delete() - s3_client.delete_bucket(Bucket=bucket_name) +def cleanup(bucket_name, availability_zone=None, client=s3_client): + if availability_zone is not None: + bucket_name = bucket_name+f"--{availability_zone}--x-s3" + + objects = client.list_objects_v2(Bucket=bucket_name)["Contents"] + objects = list(map(lambda x: {"Key": x["Key"]}, objects)) + client.delete_objects(Bucket=bucket_name, Delete={"Objects": objects}) + client.delete_bucket(Bucket=bucket_name) print(f"Bucket {bucket_name} deleted", file=sys.stderr) if args.action == 'init': try: print(BUCKET_NAME_BASE + " " + PUBLIC_BUCKET_NAME + " initializing...") - # TODO we cannot use the client across region - create_bucket_with_lifecycle("use1-az4") + create_bucket_with_lifecycle("use1-az4", s3_client_east1) create_bucket_with_lifecycle("usw2-az1") - # create_bucket_with_lifecycle() - # create_bucket_with_public_object() + create_bucket_with_lifecycle() + create_bucket_with_public_object() if os.environ.get('CRT_S3_TEST_BUCKET_NAME') != BUCKET_NAME_BASE: print( f"* Please set the environment variable $CRT_S3_TEST_BUCKET_NAME to {BUCKET_NAME_BASE} before running the tests.") except Exception as e: print(e) - # try: - # # Try to clean up the bucket created, when initialization failed. - # # cleanup(BUCKET_NAME_BASE) - # # cleanup(PUBLIC_BUCKET_NAME) - # except Exception as e2: - # exit(-1) + try: + # Try to clean up the bucket created, when initialization failed. + cleanup(BUCKET_NAME_BASE, "use1-az4", s3_client_east1) + cleanup(BUCKET_NAME_BASE, "usw2-az1") + cleanup(BUCKET_NAME_BASE) + cleanup(PUBLIC_BUCKET_NAME) + except Exception as e2: + exit(-1) exit(-1) elif args.action == 'clean': if "CRT_S3_TEST_BUCKET_NAME" not in os.environ and args.bucket_name is None: print("Set the environment variable CRT_S3_TEST_BUCKET_NAME before clean up, or pass in bucket_name as argument.") exit(-1) + cleanup(BUCKET_NAME_BASE, "use1-az4", s3_client_east1) + cleanup(BUCKET_NAME_BASE, "usw2-az1") cleanup(BUCKET_NAME_BASE) cleanup(PUBLIC_BUCKET_NAME)