Skip to content

Commit

Permalink
make the helper work for s3express
Browse files Browse the repository at this point in the history
  • Loading branch information
@TingDaoK
TingDaoK committed Dec 2, 2023
1 parent 5ae133c commit 57df345
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 50 deletions.
51 changes: 26 additions & 25 deletions tests/s3_s3express_client_test.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,6 @@ static int s_s3express_client_put_test_helper(struct aws_allocator *allocator, s

struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1");

char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com";
struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint);
struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-test");

struct aws_s3_client_config client_config = {
Expand All @@ -243,13 +241,23 @@ static int s_s3express_client_put_test_helper(struct aws_allocator *allocator, s
struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);

ASSERT_SUCCESS(s_s3express_put_object_request(
allocator, client, content_length, &tester, host_cursor, key_cursor, region_cursor));
allocator,
client,
content_length,
&tester,
g_test_s3express_bucket_use1_az4_endpoint,
key_cursor,
region_cursor));

struct aws_byte_cursor west2_region_cursor = aws_byte_cursor_from_c_str("us-west-2");
char west2_endpoint[] = "crts-west2--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com";
struct aws_byte_cursor west2_host_cursor = aws_byte_cursor_from_c_str(west2_endpoint);
ASSERT_SUCCESS(s_s3express_put_object_request(
allocator, client, content_length, &tester, west2_host_cursor, key_cursor, west2_region_cursor));
allocator,
client,
content_length,
&tester,
g_test_s3express_bucket_usw2_az1_endpoint,
key_cursor,
west2_region_cursor));

aws_s3_client_release(client);
aws_s3_tester_clean_up(&tester);
Expand Down Expand Up @@ -280,13 +288,9 @@ TEST_CASE(s3express_client_put_object_multipart_multiple) {

struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1");

char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com";
struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint);
struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-test");

struct aws_byte_cursor west2_region_cursor = aws_byte_cursor_from_c_str("us-west-2");
char west2_endpoint[] = "crts-west2--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com";
struct aws_byte_cursor west2_host_cursor = aws_byte_cursor_from_c_str(west2_endpoint);

struct aws_s3_client_config client_config = {
.part_size = MB_TO_BYTES(5),
Expand All @@ -302,11 +306,11 @@ TEST_CASE(s3express_client_put_object_multipart_multiple) {
input_streams[i] = aws_s3_test_input_stream_new(allocator, MB_TO_BYTES(10));

struct aws_byte_cursor request_region = region_cursor;
struct aws_byte_cursor request_host = host_cursor;
struct aws_byte_cursor request_host = g_test_s3express_bucket_use1_az4_endpoint;
if (i % 2 == 0) {
/* Make half of request to east1 and rest half to west2 */
request_region = west2_region_cursor;
request_host = west2_host_cursor;
request_host = g_test_s3express_bucket_usw2_az1_endpoint;
}

struct aws_http_message *message = aws_s3_test_put_object_request_new(
Expand Down Expand Up @@ -401,8 +405,6 @@ TEST_CASE(s3express_client_put_object_long_running_session_refresh) {

struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1");

char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com";
struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint);
struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-test");

struct aws_s3_client_config client_config = {
Expand All @@ -421,7 +423,12 @@ TEST_CASE(s3express_client_put_object_long_running_session_refresh) {
struct aws_input_stream *upload_stream = aws_s3_test_input_stream_new(allocator, MB_TO_BYTES(10));

struct aws_http_message *message = aws_s3_test_put_object_request_new(
allocator, &host_cursor, key_cursor, g_test_body_content_type, upload_stream, 0);
allocator,
&g_test_s3express_bucket_use1_az4_endpoint,
key_cursor,
g_test_body_content_type,
upload_stream,
0);
struct aws_s3_meta_request_options options;
AWS_ZERO_STRUCT(options);
options.type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT;
Expand Down Expand Up @@ -473,10 +480,6 @@ TEST_CASE(s3express_client_get_object) {

struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1");

char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com";
struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint);
struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-download-10MB");

struct aws_s3_client_config client_config = {
.part_size = MB_TO_BYTES(5),
.enable_s3express = true,
Expand All @@ -487,7 +490,8 @@ TEST_CASE(s3express_client_get_object) {

struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);

struct aws_http_message *message = aws_s3_test_get_object_request_new(allocator, host_cursor, key_cursor);
struct aws_http_message *message = aws_s3_test_get_object_request_new(
allocator, g_test_s3express_bucket_use1_az4_endpoint, g_pre_existing_object_10MB);

struct aws_s3_meta_request_options options;
AWS_ZERO_STRUCT(options);
Expand Down Expand Up @@ -532,10 +536,6 @@ TEST_CASE(s3express_client_get_object_multiple) {

struct aws_byte_cursor region_cursor = aws_byte_cursor_from_c_str("us-east-1");

char endpoint[] = "crts-east1--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com";
struct aws_byte_cursor host_cursor = aws_byte_cursor_from_c_str(endpoint);
struct aws_byte_cursor key_cursor = aws_byte_cursor_from_c_str("/crt-download-10MB");

struct aws_s3_client_config client_config = {
.part_size = MB_TO_BYTES(5),
.enable_s3express = true,
Expand All @@ -548,7 +548,8 @@ TEST_CASE(s3express_client_get_object_multiple) {

for (size_t i = 0; i < num_meta_requests; ++i) {

struct aws_http_message *message = aws_s3_test_get_object_request_new(allocator, host_cursor, key_cursor);
struct aws_http_message *message = aws_s3_test_get_object_request_new(
allocator, g_test_s3express_bucket_use1_az4_endpoint, g_pre_existing_object_10MB);

struct aws_s3_meta_request_options options;
AWS_ZERO_STRUCT(options);
Expand Down
56 changes: 31 additions & 25 deletions tests/test_helper/test_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@

print(boto3.__version__)

REGION = 'us-east-1'
REGION = 'us-west-2'
REGION_EAST_1 = 'us-east-1'
s3 = boto3.resource('s3')
s3_client = boto3.client('s3', region_name=REGION)
s3_client_east1 = boto3.client('s3', region_name=REGION_EAST_1)


s3_control_client = boto3.client('s3control')
Expand Down Expand Up @@ -50,9 +52,9 @@ def create_bytes(size):
return bytearray([1] * size)


def put_pre_existing_objects(size, keyname, bucket=BUCKET_NAME_BASE, sse=None, public_read=False):
def put_pre_existing_objects(size, keyname, bucket=BUCKET_NAME_BASE, sse=None, public_read=False, client=s3_client):
if size == 0:
s3_client.put_object(Bucket=bucket, Key=keyname)
client.put_object(Bucket=bucket, Key=keyname)
print(f"Object {keyname} uploaded")
return

Expand All @@ -71,7 +73,7 @@ def put_pre_existing_objects(size, keyname, bucket=BUCKET_NAME_BASE, sse=None, p
if public_read:
args['ACL'] = 'public-read'
try:
s3_client.put_object(**args)
client.put_object(**args)
except botocore.exceptions.ClientError as e:
print(f"Object {keyname} failed to upload, with exception: {e}")
if public_read and e.response['Error']['Code'] == 'AccessDenied':
Expand All @@ -80,7 +82,7 @@ def put_pre_existing_objects(size, keyname, bucket=BUCKET_NAME_BASE, sse=None, p
print(f"Object {keyname} uploaded")


def create_bucket_with_lifecycle(availability_zone=None):
def create_bucket_with_lifecycle(availability_zone=None, client=s3_client):
try:
# Create the bucket. This returns an error if the bucket already exists.

Expand All @@ -100,13 +102,10 @@ def create_bucket_with_lifecycle(availability_zone=None):
bucket_config = {'LocationConstraint': REGION}
bucket_name = BUCKET_NAME_BASE

print(bucket_name)
print(bucket_config)

s3_client.create_bucket(
client.create_bucket(
Bucket=bucket_name, CreateBucketConfiguration=bucket_config)
if availability_zone is None:
s3_client.put_bucket_lifecycle_configuration(
client.put_bucket_lifecycle_configuration(
Bucket=bucket_name,
LifecycleConfiguration={
'Rules': [
Expand All @@ -132,7 +131,7 @@ def create_bucket_with_lifecycle(availability_zone=None):
print(f"Bucket {bucket_name} created", file=sys.stderr)

put_pre_existing_objects(
10*MB, 'pre-existing-10MB', bucket=bucket_name)
10*MB, 'pre-existing-10MB', bucket=bucket_name, client=client)

if availability_zone is None:
put_pre_existing_objects(
Expand Down Expand Up @@ -190,36 +189,43 @@ def create_bucket_with_public_object():
raise e


def cleanup(bucket_name):
bucket = s3.Bucket(bucket_name)
bucket.objects.all().delete()
s3_client.delete_bucket(Bucket=bucket_name)
def cleanup(bucket_name, availability_zone=None, client=s3_client):
if availability_zone is not None:
bucket_name = bucket_name+f"--{availability_zone}--x-s3"

objects = client.list_objects_v2(Bucket=bucket_name)["Contents"]
objects = list(map(lambda x: {"Key": x["Key"]}, objects))
client.delete_objects(Bucket=bucket_name, Delete={"Objects": objects})
client.delete_bucket(Bucket=bucket_name)
print(f"Bucket {bucket_name} deleted", file=sys.stderr)


if args.action == 'init':
try:
print(BUCKET_NAME_BASE + " " + PUBLIC_BUCKET_NAME + " initializing...")
# TODO we cannot use the client across region
create_bucket_with_lifecycle("use1-az4")
create_bucket_with_lifecycle("use1-az4", s3_client_east1)
create_bucket_with_lifecycle("usw2-az1")
# create_bucket_with_lifecycle()
# create_bucket_with_public_object()
create_bucket_with_lifecycle()
create_bucket_with_public_object()
if os.environ.get('CRT_S3_TEST_BUCKET_NAME') != BUCKET_NAME_BASE:
print(
f"* Please set the environment variable $CRT_S3_TEST_BUCKET_NAME to {BUCKET_NAME_BASE} before running the tests.")
except Exception as e:
print(e)
# try:
# # Try to clean up the bucket created, when initialization failed.
# # cleanup(BUCKET_NAME_BASE)
# # cleanup(PUBLIC_BUCKET_NAME)
# except Exception as e2:
# exit(-1)
try:
# Try to clean up the bucket created, when initialization failed.
cleanup(BUCKET_NAME_BASE, "use1-az4", s3_client_east1)
cleanup(BUCKET_NAME_BASE, "usw2-az1")
cleanup(BUCKET_NAME_BASE)
cleanup(PUBLIC_BUCKET_NAME)
except Exception as e2:
exit(-1)
exit(-1)
elif args.action == 'clean':
if "CRT_S3_TEST_BUCKET_NAME" not in os.environ and args.bucket_name is None:
print("Set the environment variable CRT_S3_TEST_BUCKET_NAME before clean up, or pass in bucket_name as argument.")
exit(-1)
cleanup(BUCKET_NAME_BASE, "use1-az4", s3_client_east1)
cleanup(BUCKET_NAME_BASE, "usw2-az1")
cleanup(BUCKET_NAME_BASE)
cleanup(PUBLIC_BUCKET_NAME)

0 comments on commit 57df345

Please sign in to comment.