Skip to content

Commit

Permalink
Add tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@waahm7
waahm7 committed Dec 3, 2023
1 parent 6913cd3 commit c3912f1
Show file tree
Hide file tree
Showing 3 changed files with 141 additions and 25 deletions.
7 changes: 3 additions & 4 deletions source/s3_meta_request.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,7 @@ static int s_meta_request_get_response_headers_checksum_callback(
continue;
}
const struct aws_byte_cursor *algorithm_header_name = aws_get_http_header_name_from_algorithm(i);
// waahm7: TODO need information here to find out if this object was MPU or Not
if (aws_http_headers_has(headers, *algorithm_header_name) &&
!aws_http_headers_has(headers, g_mp_parts_count_header_name)) {
if (aws_http_headers_has(headers, *algorithm_header_name)) {
struct aws_byte_cursor header_sum;
aws_http_headers_get(headers, *algorithm_header_name, &header_sum);
size_t encoded_len = 0;
Expand Down Expand Up @@ -1285,7 +1283,7 @@ static int s_s3_meta_request_headers_block_done(
uint64_t content_length;
if (!aws_s3_parse_content_length_response_header(
request->allocator, request->send_data.response_headers, &content_length) &&
content_length > meta_request->part_size) {
content_length != meta_request->part_size) {
return aws_raise_error(AWS_ERROR_S3_PART_TOO_LARGE_FOR_GET_PART);
}
}
Expand Down Expand Up @@ -1536,6 +1534,7 @@ void aws_s3_meta_request_send_request_finish_default(

/* If the request failed due to an invalid (ie: unrecoverable) response status, or the meta request already
* has a result, then make sure that this request isn't retried. */
// TODO: expected error, don't log at error
if (error_code == AWS_ERROR_S3_INVALID_RESPONSE_STATUS ||
error_code == AWS_ERROR_S3_PART_TOO_LARGE_FOR_GET_PART ||
error_code == AWS_ERROR_S3_NON_RECOVERABLE_ASYNC_ERROR || meta_request_finishing) {
Expand Down
3 changes: 2 additions & 1 deletion tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ add_net_test_case(test_s3_put_object_async_no_content_length_1part)
add_net_test_case(test_s3_put_object_async_no_content_length_empty_part2)
add_net_test_case(test_s3_put_object_async_no_content_length_2parts)
add_net_test_case(test_s3_put_object_async_fail_reading)
add_net_test_case(test_s3_download_multipart_get_single_part_upload)
add_net_test_case(test_s3_download_single_part_file_with_checksum)
add_net_test_case(test_s3_download_multi_part_file_with_checksum)

if(ENABLE_MRAP_TESTS)
add_net_test_case(test_s3_get_object_less_than_part_size_mrap)
Expand Down
156 changes: 136 additions & 20 deletions tests/s3_data_plane_tests.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3595,10 +3595,97 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *alloc
return 0;
}

AWS_TEST_CASE(test_s3_download_multipart_get_single_part_upload, s_test_s3_download_multipart_get_single_part_upload)
static int s_test_s3_download_multipart_get_single_part_upload(struct aws_allocator *allocator, void *ctx) {
// waahm7
AWS_TEST_CASE(test_s3_download_single_part_file_with_checksum, s_test_s3_download_single_part_file_with_checksum)
static int s_test_s3_download_single_part_file_with_checksum(struct aws_allocator *allocator, void *ctx) {
(void)ctx;

/* Upload the file */
struct aws_s3_tester tester;
ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
struct aws_s3_tester_client_options client_options = {
.part_size = MB_TO_BYTES(10),
};

struct aws_s3_client *client = NULL;
ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));

struct aws_byte_buf path_buf;
AWS_ZERO_STRUCT(path_buf);

ASSERT_SUCCESS(aws_s3_tester_upload_file_path_init(
allocator, &path_buf, aws_byte_cursor_from_c_str("/single-part-10Mb-CRC32.txt")));

struct aws_byte_cursor object_path = aws_byte_cursor_from_buf(&path_buf);

struct aws_s3_tester_meta_request_options put_options = {
.allocator = allocator,
.meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
.client = client,
.checksum_algorithm = AWS_SCA_CRC32,
.put_options =
{
.object_size_mb = 10,
.object_path_override = object_path,
},
};

ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &put_options, NULL));
client = aws_s3_client_release(client);
tester.bound_to_client = false;

/*** GET FILE with part_size < file_size ***/
client_options.part_size = MB_TO_BYTES(3);

ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));

struct aws_s3_tester_meta_request_options get_options = {
.allocator = allocator,
.meta_request_type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT,
.validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS,
.client = client,
.expected_validate_checksum_alg = AWS_SCA_CRC32,
.validate_get_response_checksum = true,
.get_options =
{
.object_path = object_path,
},
.finish_callback = s_s3_test_validate_checksum,
};

ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, NULL));
client = aws_s3_client_release(client);
tester.bound_to_client = false;

/*** GET FILE with part_size > file_size ***/
client_options.part_size = MB_TO_BYTES(20);

ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));
get_options.client = client;
ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, NULL));
client = aws_s3_client_release(client);
tester.bound_to_client = false;

/*** GET FILE with part_size = file_size ***/
client_options.part_size = MB_TO_BYTES(10);

ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));
get_options.client = client;
ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, NULL));
client = aws_s3_client_release(client);
tester.bound_to_client = false;

aws_byte_buf_clean_up(&path_buf);
aws_s3_tester_clean_up(&tester);

return 0;
}

AWS_TEST_CASE(test_s3_download_multi_part_file_with_checksum, s_test_s3_download_multi_part_file_with_checksum)
static int s_test_s3_download_multi_part_file_with_checksum(struct aws_allocator *allocator, void *ctx) {
(void)ctx;

/* Upload the file */
struct aws_s3_tester tester;
ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
struct aws_s3_tester_client_options client_options = {
Expand All @@ -3608,27 +3695,34 @@ static int s_test_s3_download_multipart_get_single_part_upload(struct aws_alloca
struct aws_s3_client *client = NULL;
ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));

// TODO: upload this file or use one from our test
struct aws_byte_cursor object_path = aws_byte_cursor_from_c_str("/upload/prefix/round_trip/test_mpu_fc.txt");
struct aws_byte_buf path_buf;
AWS_ZERO_STRUCT(path_buf);

// struct aws_s3_tester_meta_request_options put_options = {
// .allocator = allocator,
// .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
// .client = client,
// .checksum_algorithm = AWS_SCA_CRC32,
// .validate_get_response_checksum = false,
ASSERT_SUCCESS(aws_s3_tester_upload_file_path_init(
allocator, &path_buf, aws_byte_cursor_from_c_str("/multi-part-10Mb-CRC32.txt")));

// .put_options =
// {
// .object_size_mb = 20,
// .object_path_override = object_path,
// .ensure_multipart = true,
// },
// };
struct aws_byte_cursor object_path = aws_byte_cursor_from_buf(&path_buf);

struct aws_s3_tester_meta_request_options put_options = {
.allocator = allocator,
.meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
.client = client,
.checksum_algorithm = AWS_SCA_CRC32,
.put_options =
{
.object_size_mb = 10,
.object_path_override = object_path,
},
};

// ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &put_options, NULL));
client = aws_s3_client_release(client);
tester.bound_to_client = false;

/*** GET FILE ***/
/*** GET FILE with part_size < first_part_size ***/
client_options.part_size = MB_TO_BYTES(3);

ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));

struct aws_s3_tester_meta_request_options get_options = {
.allocator = allocator,
Expand All @@ -3641,12 +3735,34 @@ static int s_test_s3_download_multipart_get_single_part_upload(struct aws_alloca
{
.object_path = object_path,
},
.finish_callback = s_s3_test_validate_checksum,
};

// ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, NULL));
client = aws_s3_client_release(client);
tester.bound_to_client = false;

/*** GET FILE with part_size > first_part_size ***/
// client_options.part_size = MB_TO_BYTES(7);
client_options.part_size = MB_TO_BYTES(20);

ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));
get_options.client = client;
ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, NULL));
client = aws_s3_client_release(client);
tester.bound_to_client = false;

aws_s3_client_release(client);
/*** GET FILE with part_size = first_part_size ***/
client_options.part_size = MB_TO_BYTES(5);
// client_options.part_size = MB_TO_BYTES(10);

ASSERT_SUCCESS(aws_s3_tester_client_new(&tester, &client_options, &client));
get_options.client = client;
get_options.finish_callback = s_s3_test_validate_checksum;
// ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, NULL));
client = aws_s3_client_release(client);
tester.bound_to_client = false;

aws_byte_buf_clean_up(&path_buf);
aws_s3_tester_clean_up(&tester);

return 0;
Expand Down

0 comments on commit c3912f1

Please sign in to comment.