diff --git a/.gitignore b/.gitignore index 6fbfca54f..6046e192d 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,6 @@ cmake-build* # js package locks irrelevant to the overall package's security benchmarks/benchmarks-stack/benchmarks-stack/package-lock.json benchmarks/dashboard-stack/package-lock.json + +# virtual environment +.venv/ diff --git a/include/aws/s3/private/s3_meta_request_impl.h b/include/aws/s3/private/s3_meta_request_impl.h index bde40ebc4..024680c8f 100644 --- a/include/aws/s3/private/s3_meta_request_impl.h +++ b/include/aws/s3/private/s3_meta_request_impl.h @@ -350,6 +350,14 @@ void aws_s3_meta_request_init_signing_date_time_default( struct aws_s3_meta_request *meta_request, struct aws_date_time *date_time); +AWS_S3_API +void aws_s3_meta_request_sign_request_default_impl( + struct aws_s3_meta_request *meta_request, + struct aws_s3_request *request, + aws_signing_complete_fn *on_signing_complete, + void *user_data, + bool disable_s3_express_signing); + AWS_S3_API void aws_s3_meta_request_sign_request_default( struct aws_s3_meta_request *meta_request, diff --git a/include/aws/s3/s3_client.h b/include/aws/s3/s3_client.h index fbf97268c..c765d79da 100644 --- a/include/aws/s3/s3_client.h +++ b/include/aws/s3/s3_client.h @@ -82,6 +82,7 @@ enum aws_s3_meta_request_type { * - only {bucket}/{key} format is supported for source and passing arn as * source will not work * - source bucket is assumed to be in the same region as dest + * - source bucket and dest bucket must both be either directory buckets or regular buckets. */ AWS_S3_META_REQUEST_TYPE_COPY_OBJECT, diff --git a/source/s3_client.c b/source/s3_client.c index 83509a339..c1f2fcf75 100644 --- a/source/s3_client.c +++ b/source/s3_client.c @@ -346,7 +346,7 @@ struct aws_s3_client *aws_s3_client_new( client->buffer_pool = aws_s3_buffer_pool_new(allocator, part_size, mem_limit); if (client->buffer_pool == NULL) { - goto on_early_fail; + goto on_error; } struct aws_s3_buffer_pool_usage_stats pool_usage = aws_s3_buffer_pool_get_usage(client->buffer_pool); @@ -357,7 +357,7 @@ struct aws_s3_client *aws_s3_client_new( "Cannot create client from client_config; configured max part size should not exceed memory limit." "size."); aws_raise_error(AWS_ERROR_S3_INVALID_MEMORY_LIMIT_CONFIG); - goto on_early_fail; + goto on_error; } client->vtable = &s_s3_client_default_vtable; @@ -365,7 +365,7 @@ struct aws_s3_client *aws_s3_client_new( aws_ref_count_init(&client->ref_count, client, (aws_simple_completion_callback *)s_s3_client_start_destroy); if (aws_mutex_init(&client->synced_data.lock) != AWS_OP_SUCCESS) { - goto on_early_fail; + goto on_error; } aws_linked_list_init(&client->synced_data.pending_meta_request_work); @@ -488,6 +488,44 @@ struct aws_s3_client *aws_s3_client_new( } } + client->num_network_interface_names = client_config->num_network_interface_names; + if (client_config->num_network_interface_names > 0) { + AWS_LOGF_DEBUG( + AWS_LS_S3_CLIENT, + "id=%p Client received network interface names array with length %zu.", + (void *)client, + client->num_network_interface_names); + aws_array_list_init_dynamic( + &client->network_interface_names, + client->allocator, + client_config->num_network_interface_names, + sizeof(struct aws_string *)); + client->network_interface_names_cursor_array = aws_mem_calloc( + client->allocator, client_config->num_network_interface_names, sizeof(struct aws_byte_cursor)); + for (size_t i = 0; i < client_config->num_network_interface_names; i++) { + struct aws_byte_cursor interface_name = client_config->network_interface_names_array[i]; + struct aws_string *interface_name_str = aws_string_new_from_cursor(client->allocator, &interface_name); + aws_array_list_push_back(&client->network_interface_names, &interface_name_str); + if (aws_is_network_interface_name_valid(aws_string_c_str(interface_name_str)) == false) { + AWS_LOGF_ERROR( + AWS_LS_S3_CLIENT, + "id=%p network_interface_names_array[%zu]=" PRInSTR " is not valid.", + (void *)client, + i, + AWS_BYTE_CURSOR_PRI(interface_name)); + aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + goto on_error; + } + client->network_interface_names_cursor_array[i] = aws_byte_cursor_from_string(interface_name_str); + AWS_LOGF_DEBUG( + AWS_LS_S3_CLIENT, + "id=%p network_interface_names_array[%zu]=" PRInSTR "", + (void *)client, + i, + AWS_BYTE_CURSOR_PRI(client->network_interface_names_cursor_array[i])); + } + } + /* Set up body streaming ELG */ { uint16_t num_event_loops = @@ -579,34 +617,6 @@ struct aws_s3_client *aws_s3_client_new( *((bool *)&client->enable_read_backpressure) = client_config->enable_read_backpressure; *((size_t *)&client->initial_read_window) = client_config->initial_read_window; - client->num_network_interface_names = client_config->num_network_interface_names; - if (client_config->num_network_interface_names > 0) { - AWS_LOGF_DEBUG( - AWS_LS_S3_CLIENT, - "id=%p Client received network interface names array with length %zu.", - (void *)client, - client->num_network_interface_names); - aws_array_list_init_dynamic( - &client->network_interface_names, - client->allocator, - client_config->num_network_interface_names, - sizeof(struct aws_string *)); - client->network_interface_names_cursor_array = aws_mem_calloc( - client->allocator, client_config->num_network_interface_names, sizeof(struct aws_byte_cursor)); - for (size_t i = 0; i < client_config->num_network_interface_names; i++) { - struct aws_byte_cursor interface_name = client_config->network_interface_names_array[i]; - struct aws_string *interface_name_str = aws_string_new_from_cursor(client->allocator, &interface_name); - aws_array_list_push_back(&client->network_interface_names, &interface_name_str); - client->network_interface_names_cursor_array[i] = aws_byte_cursor_from_string(interface_name_str); - AWS_LOGF_DEBUG( - AWS_LS_S3_CLIENT, - "id=%p network_interface_names_array[%zu]=" PRInSTR "", - (void *)client, - i, - AWS_BYTE_CURSOR_PRI(client->network_interface_names_cursor_array[i])); - } - } - return client; on_error: @@ -628,10 +638,22 @@ struct aws_s3_client *aws_s3_client_new( aws_mem_release(client->allocator, client->proxy_ev_settings); aws_mem_release(client->allocator, client->tcp_keep_alive_options); - aws_event_loop_group_release(client->client_bootstrap->event_loop_group); + if (client->client_bootstrap != NULL) { + aws_event_loop_group_release(client->client_bootstrap->event_loop_group); + } aws_client_bootstrap_release(client->client_bootstrap); aws_mutex_clean_up(&client->synced_data.lock); -on_early_fail: + + aws_mem_release(client->allocator, client->network_interface_names_cursor_array); + for (size_t i = 0; i < aws_array_list_length(&client->network_interface_names); i++) { + struct aws_string *interface_name = NULL; + aws_array_list_get_at(&client->network_interface_names, &interface_name, i); + aws_string_destroy(interface_name); + } + + aws_array_list_clean_up(&client->network_interface_names); + aws_s3_buffer_pool_destroy(client->buffer_pool); + aws_mem_release(client->allocator, client); return NULL; } diff --git a/source/s3_copy_object.c b/source/s3_copy_object.c index 86dec71a4..432a6902a 100644 --- a/source/s3_copy_object.c +++ b/source/s3_copy_object.c @@ -37,12 +37,18 @@ static void s_s3_copy_object_request_finished( struct aws_s3_request *request, int error_code); +static void s_s3_copy_object_sign_request( + struct aws_s3_meta_request *meta_request, + struct aws_s3_request *request, + aws_signing_complete_fn *on_signing_complete, + void *user_data); + static struct aws_s3_meta_request_vtable s_s3_copy_object_vtable = { .update = s_s3_copy_object_update, .send_request_finish = aws_s3_meta_request_send_request_finish_default, .prepare_request = s_s3_copy_object_prepare_request, .init_signing_date_time = aws_s3_meta_request_init_signing_date_time_default, - .sign_request = aws_s3_meta_request_sign_request_default, + .sign_request = s_s3_copy_object_sign_request, .finished_request = s_s3_copy_object_request_finished, .destroy = s_s3_meta_request_copy_object_destroy, .finish = aws_s3_meta_request_finish_default, @@ -796,3 +802,21 @@ static void s_s3_copy_object_request_finished( aws_s3_meta_request_unlock_synced_data(meta_request); } + +static void s_s3_copy_object_sign_request( + struct aws_s3_meta_request *meta_request, + struct aws_s3_request *request, + aws_signing_complete_fn *on_signing_complete, + void *user_data) { + + /** + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html + * For CopyObject and UploadPartCopy, the request has to be signed with IAM credentials for directory buckets. + * Disable S3 express signing for those types. + */ + bool disable_s3_express_signing = request->request_tag == AWS_S3_COPY_OBJECT_REQUEST_TAG_BYPASS || + request->request_tag == AWS_S3_COPY_OBJECT_REQUEST_TAG_MULTIPART_COPY; + aws_s3_meta_request_sign_request_default_impl( + meta_request, request, on_signing_complete, user_data, disable_s3_express_signing); +} diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c index ccafe63ca..a78034c4c 100644 --- a/source/s3_meta_request.c +++ b/source/s3_meta_request.c @@ -900,12 +900,12 @@ static int s_meta_request_resolve_signing_config( return AWS_OP_SUCCESS; } -/* Handles signing a message for the caller. */ -void aws_s3_meta_request_sign_request_default( +void aws_s3_meta_request_sign_request_default_impl( struct aws_s3_meta_request *meta_request, struct aws_s3_request *request, aws_signing_complete_fn *on_signing_complete, - void *user_data) { + void *user_data, + bool disable_s3_express_signing) { AWS_PRECONDITION(meta_request); AWS_PRECONDITION(request); AWS_PRECONDITION(on_signing_complete); @@ -947,7 +947,7 @@ void aws_s3_meta_request_sign_request_default( return; } - if (signing_config.algorithm == AWS_SIGNING_ALGORITHM_V4_S3EXPRESS) { + if (signing_config.algorithm == AWS_SIGNING_ALGORITHM_V4_S3EXPRESS && !disable_s3_express_signing) { /* Fetch credentials from S3 Express provider. */ struct aws_get_s3express_credentials_user_data *context = aws_mem_calloc(meta_request->allocator, 1, sizeof(struct aws_get_s3express_credentials_user_data)); @@ -998,6 +998,9 @@ void aws_s3_meta_request_sign_request_default( } } else { /* Regular signing. */ + if (disable_s3_express_signing) { + signing_config.algorithm = AWS_SIGNING_ALGORITHM_V4; + } s_s3_meta_request_init_signing_date_time(meta_request, &signing_config.date); if (aws_sign_request_aws( meta_request->allocator, @@ -1015,6 +1018,15 @@ void aws_s3_meta_request_sign_request_default( } } +/* Handles signing a message for the caller. */ +void aws_s3_meta_request_sign_request_default( + struct aws_s3_meta_request *meta_request, + struct aws_s3_request *request, + aws_signing_complete_fn *on_signing_complete, + void *user_data) { + aws_s3_meta_request_sign_request_default_impl(meta_request, request, on_signing_complete, user_data, false); +} + /* Handle the signing result */ static void s_s3_meta_request_request_on_signed( struct aws_signing_result *signing_result, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4004408d7..52a357a69 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,6 +27,8 @@ add_test_case(test_s3_abort_multipart_upload_message_new) add_net_test_case(test_s3_client_create_destroy) add_net_test_case(test_s3_client_create_error) +add_net_test_case(test_s3_client_create_error_with_invalid_memory_limit_config) +add_net_test_case(test_s3_client_create_error_with_invalid_network_interface) add_net_test_case(test_s3_client_monitoring_options_override) add_net_test_case(test_s3_client_proxy_ev_settings_override) add_net_test_case(test_s3_client_tcp_keep_alive_options_override) @@ -352,6 +354,8 @@ add_net_test_case(s3express_client_put_object_long_running_session_refresh) add_net_test_case(s3express_client_get_object) add_net_test_case(s3express_client_get_object_multiple) add_net_test_case(s3express_client_get_object_create_session_error) +add_net_test_case(s3express_client_copy_object) +add_net_test_case(s3express_client_copy_object_multipart) add_net_test_case(meta_request_auto_ranged_get_new_error_handling) add_net_test_case(meta_request_auto_ranged_put_new_error_handling) diff --git a/tests/s3_data_plane_tests.c b/tests/s3_data_plane_tests.c index ab01e395b..4154f5150 100644 --- a/tests/s3_data_plane_tests.c +++ b/tests/s3_data_plane_tests.c @@ -77,6 +77,61 @@ static int s_test_s3_client_create_error(struct aws_allocator *allocator, void * return 0; } +AWS_TEST_CASE( + test_s3_client_create_error_with_invalid_memory_limit_config, + s_test_s3_client_create_error_with_invalid_memory_limit_config) +static int s_test_s3_client_create_error_with_invalid_memory_limit_config(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_client_config client_config; + AWS_ZERO_STRUCT(client_config); + ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION)); + + client_config.memory_limit_in_bytes = 100; + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + ASSERT_TRUE(client == NULL); + client_config.memory_limit_in_bytes = GB_TO_BYTES(1); + client_config.max_part_size = GB_TO_BYTES(2); + client = aws_s3_client_new(allocator, &client_config); + ASSERT_TRUE(client == NULL); + + tester.bound_to_client = false; + aws_s3_tester_clean_up(&tester); + return 0; +} + +AWS_TEST_CASE( + test_s3_client_create_error_with_invalid_network_interface, + s_test_s3_client_create_error_with_invalid_network_interface) +static int s_test_s3_client_create_error_with_invalid_network_interface(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_client_config client_config; + AWS_ZERO_STRUCT(client_config); + ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION)); + + struct aws_byte_cursor *interface_names_array = aws_mem_calloc(allocator, 1, sizeof(struct aws_byte_cursor)); + interface_names_array[0] = aws_byte_cursor_from_c_str("invalid-nic"); + client_config.network_interface_names_array = interface_names_array; + client_config.num_network_interface_names = 1; + + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + ASSERT_TRUE(client == NULL); + tester.bound_to_client = false; + + aws_s3_tester_clean_up(&tester); + aws_mem_release(allocator, interface_names_array); + return 0; +} + AWS_TEST_CASE(test_s3_client_monitoring_options_override, s_test_s3_client_monitoring_options_override) static int s_test_s3_client_monitoring_options_override(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -6305,167 +6360,15 @@ static int s_test_s3_invalid_empty_file_with_range(struct aws_allocator *allocat return 0; } -static const struct aws_byte_cursor g_x_amz_copy_source_name = - AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"); - -struct aws_http_message *copy_object_request_new( - struct aws_allocator *allocator, - struct aws_byte_cursor x_amz_source, - struct aws_byte_cursor endpoint, - struct aws_byte_cursor destination_key) { - - AWS_PRECONDITION(allocator); - - struct aws_http_message *message = aws_http_message_new_request(allocator); - - if (message == NULL) { - return NULL; - } - - /* the URI path is / followed by the key */ - char destination_path[1024]; - snprintf(destination_path, sizeof(destination_path), "/%.*s", (int)destination_key.len, destination_key.ptr); - struct aws_byte_cursor unencoded_destination_path = aws_byte_cursor_from_c_str(destination_path); - struct aws_byte_buf copy_destination_path_encoded; - aws_byte_buf_init(©_destination_path_encoded, allocator, 1024); - aws_byte_buf_append_encoding_uri_path(©_destination_path_encoded, &unencoded_destination_path); - if (aws_http_message_set_request_path(message, aws_byte_cursor_from_buf(©_destination_path_encoded))) { - goto error_clean_up_message; - } - - struct aws_http_header host_header = {.name = g_host_header_name, .value = endpoint}; - if (aws_http_message_add_header(message, host_header)) { - goto error_clean_up_message; - } - - struct aws_byte_buf copy_source_value_encoded; - aws_byte_buf_init(©_source_value_encoded, allocator, 1024); - aws_byte_buf_append_encoding_uri_path(©_source_value_encoded, &x_amz_source); - - struct aws_http_header copy_source_header = { - .name = g_x_amz_copy_source_name, - .value = aws_byte_cursor_from_buf(©_source_value_encoded), - }; - - if (aws_http_message_add_header(message, copy_source_header)) { - goto error_clean_up_message; - } - - if (aws_http_message_set_request_method(message, aws_http_method_put)) { - goto error_clean_up_message; - } - - aws_byte_buf_clean_up(©_source_value_encoded); - aws_byte_buf_clean_up(©_destination_path_encoded); - return message; - -error_clean_up_message: - - aws_byte_buf_clean_up(©_source_value_encoded); - aws_byte_buf_clean_up(©_destination_path_encoded); - if (message != NULL) { - aws_http_message_release(message); - message = NULL; - } - - return NULL; -} - -struct copy_object_test_data { - struct aws_mutex mutex; - struct aws_condition_variable c_var; - bool execution_completed; - bool headers_callback_was_invoked; - int meta_request_error_code; - int response_status_code; - uint64_t progress_callback_content_length; - uint64_t progress_callback_total_bytes_transferred; -}; - -static void s_copy_object_meta_request_finish( - struct aws_s3_meta_request *meta_request, - const struct aws_s3_meta_request_result *meta_request_result, - void *user_data) { - - (void)meta_request; - - struct copy_object_test_data *test_data = user_data; - - /* if error response body is available, dump it to test result to help investigation of failed tests */ - if (meta_request_result->error_response_body != NULL && meta_request_result->error_response_body->len > 0) { - AWS_LOGF_ERROR( - AWS_LS_S3_GENERAL, - "Response error body: %.*s", - (int)meta_request_result->error_response_body->len, - meta_request_result->error_response_body->buffer); - } - - aws_mutex_lock(&test_data->mutex); - test_data->meta_request_error_code = meta_request_result->error_code; - test_data->response_status_code = meta_request_result->response_status; - test_data->execution_completed = true; - aws_mutex_unlock(&test_data->mutex); - aws_condition_variable_notify_one(&test_data->c_var); -} - -static int s_copy_object_meta_request_headers_callback( - struct aws_s3_meta_request *meta_request, - const struct aws_http_headers *headers, - int response_status, - void *user_data) { - - (void)meta_request; - (void)headers; - (void)response_status; - - struct copy_object_test_data *test_data = user_data; - - aws_mutex_lock(&test_data->mutex); - test_data->headers_callback_was_invoked = true; - aws_mutex_unlock(&test_data->mutex); - - return AWS_OP_SUCCESS; -} - -static void s_copy_object_meta_request_progress_callback( - struct aws_s3_meta_request *meta_request, - const struct aws_s3_meta_request_progress *progress, - void *user_data) { - - (void)meta_request; - struct copy_object_test_data *test_data = user_data; - - aws_mutex_lock(&test_data->mutex); - test_data->progress_callback_content_length = progress->content_length; - test_data->progress_callback_total_bytes_transferred += progress->bytes_transferred; - aws_mutex_unlock(&test_data->mutex); -} - -static bool s_copy_test_completion_predicate(void *arg) { - struct copy_object_test_data *test_data = arg; - return test_data->execution_completed; -} - -static int s_test_s3_copy_object_from_x_amz_copy_source( +static int s_test_s3_copy_object_helper( struct aws_allocator *allocator, - struct aws_byte_cursor x_amz_copy_source, + struct aws_byte_cursor source_key, struct aws_byte_cursor destination_key, int expected_error_code, int expected_response_status, uint64_t expected_size) { - struct aws_s3_tester tester; - AWS_ZERO_STRUCT(tester); - ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); - - struct aws_s3_client_config client_config; - AWS_ZERO_STRUCT(client_config); - - ASSERT_SUCCESS(aws_s3_tester_bind_client( - &tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION | AWS_S3_TESTER_BIND_CLIENT_SIGNING)); - - struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); - + struct aws_byte_cursor source_bucket = g_test_bucket_name; struct aws_byte_cursor destination_bucket = g_test_bucket_name; char endpoint[1024]; @@ -6477,83 +6380,16 @@ static int s_test_s3_copy_object_from_x_amz_copy_source( destination_bucket.ptr, g_test_s3_region.ptr); - /* creates a CopyObject request */ - struct aws_http_message *message = - copy_object_request_new(allocator, x_amz_copy_source, aws_byte_cursor_from_c_str(endpoint), destination_key); - - struct copy_object_test_data test_data; - AWS_ZERO_STRUCT(test_data); - - test_data.c_var = (struct aws_condition_variable)AWS_CONDITION_VARIABLE_INIT; - aws_mutex_init(&test_data.mutex); - - struct aws_s3_meta_request_options meta_request_options = { - .user_data = &test_data, - .body_callback = NULL, - .signing_config = client_config.signing_config, - .finish_callback = s_copy_object_meta_request_finish, - .headers_callback = s_copy_object_meta_request_headers_callback, - .progress_callback = s_copy_object_meta_request_progress_callback, - .message = message, - .shutdown_callback = NULL, - .type = AWS_S3_META_REQUEST_TYPE_COPY_OBJECT, - }; - - struct aws_s3_meta_request *meta_request = aws_s3_client_make_meta_request(client, &meta_request_options); - ASSERT_NOT_NULL(meta_request); - - /* wait completion of the meta request */ - aws_mutex_lock(&test_data.mutex); - aws_condition_variable_wait_pred(&test_data.c_var, &test_data.mutex, s_copy_test_completion_predicate, &test_data); - aws_mutex_unlock(&test_data.mutex); - - /* assert error_code and response_status_code */ - ASSERT_INT_EQUALS(expected_error_code, test_data.meta_request_error_code); - ASSERT_INT_EQUALS(expected_response_status, test_data.response_status_code); - - /* assert that progress_callback matches the expected size*/ - if (test_data.meta_request_error_code == AWS_ERROR_SUCCESS) { - ASSERT_UINT_EQUALS(expected_size, test_data.progress_callback_total_bytes_transferred); - ASSERT_UINT_EQUALS(expected_size, test_data.progress_callback_content_length); - } - - /* assert headers callback was invoked */ - ASSERT_TRUE(test_data.headers_callback_was_invoked); - - aws_s3_meta_request_release(meta_request); - aws_mutex_clean_up(&test_data.mutex); - aws_http_message_destroy(message); - client = aws_s3_client_release(client); - - aws_s3_tester_clean_up(&tester); - - return 0; -} - -static int s_test_s3_copy_object_helper( - struct aws_allocator *allocator, - struct aws_byte_cursor source_key, - struct aws_byte_cursor destination_key, - int expected_error_code, - int expected_response_status, - uint64_t expected_size) { - - struct aws_byte_cursor source_bucket = g_test_bucket_name; - - char copy_source_value[1024]; - snprintf( - copy_source_value, - sizeof(copy_source_value), - "%.*s/%.*s", - (int)source_bucket.len, - source_bucket.ptr, - (int)source_key.len, - source_key.ptr); - - struct aws_byte_cursor x_amz_copy_source = aws_byte_cursor_from_c_str(copy_source_value); - - return s_test_s3_copy_object_from_x_amz_copy_source( - allocator, x_amz_copy_source, destination_key, expected_error_code, expected_response_status, expected_size); + return aws_test_s3_copy_object_helper( + allocator, + source_bucket, + source_key, + aws_byte_cursor_from_c_str(endpoint), + destination_key, + expected_error_code, + expected_response_status, + expected_size, + false); } AWS_TEST_CASE(test_s3_copy_small_object, s_test_s3_copy_small_object) @@ -6638,9 +6474,26 @@ static int s_test_s3_copy_source_prefixed_by_slash(struct aws_allocator *allocat source_key.ptr); struct aws_byte_cursor x_amz_copy_source = aws_byte_cursor_from_c_str(copy_source_value); + struct aws_byte_cursor destination_bucket = g_test_bucket_name; - return s_test_s3_copy_object_from_x_amz_copy_source( - allocator, x_amz_copy_source, destination_key, AWS_ERROR_SUCCESS, AWS_HTTP_STATUS_CODE_200_OK, MB_TO_BYTES(1)); + char endpoint[1024]; + snprintf( + endpoint, + sizeof(endpoint), + "%.*s.s3.%s.amazonaws.com", + (int)destination_bucket.len, + destination_bucket.ptr, + g_test_s3_region.ptr); + + return aws_test_s3_copy_object_from_x_amz_copy_source( + allocator, + x_amz_copy_source, + aws_byte_cursor_from_c_str(endpoint), + destination_key, + AWS_ERROR_SUCCESS, + AWS_HTTP_STATUS_CODE_200_OK, + MB_TO_BYTES(1), + false); } /** @@ -6668,14 +6521,26 @@ static int s_test_s3_copy_source_prefixed_by_slash_multipart(struct aws_allocato source_key.ptr); struct aws_byte_cursor x_amz_copy_source = aws_byte_cursor_from_c_str(copy_source_value); + struct aws_byte_cursor destination_bucket = g_test_bucket_name; + + char endpoint[1024]; + snprintf( + endpoint, + sizeof(endpoint), + "%.*s.s3.%s.amazonaws.com", + (int)destination_bucket.len, + destination_bucket.ptr, + g_test_s3_region.ptr); - return s_test_s3_copy_object_from_x_amz_copy_source( + return aws_test_s3_copy_object_from_x_amz_copy_source( allocator, x_amz_copy_source, + aws_byte_cursor_from_c_str(endpoint), destination_key, AWS_ERROR_SUCCESS, AWS_HTTP_STATUS_CODE_200_OK, - MB_TO_BYTES(256)); + MB_TO_BYTES(256), + false); } static int s_s3_get_object_mrap_helper(struct aws_allocator *allocator, bool multipart) { diff --git a/tests/s3_mock_server_tests.c b/tests/s3_mock_server_tests.c index 29aadda03..1bb4b5b5d 100644 --- a/tests/s3_mock_server_tests.c +++ b/tests/s3_mock_server_tests.c @@ -169,16 +169,19 @@ TEST_CASE(multipart_upload_mock_server) { TEST_CASE(multipart_upload_with_network_interface_names_mock_server) { (void)ctx; - +#if defined(AWS_OS_WINDOWS) + (void)allocator; + return AWS_OP_SKIP; +#else struct aws_s3_tester tester; ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); struct aws_byte_cursor *interface_names_array = aws_mem_calloc(allocator, 2, sizeof(struct aws_byte_cursor)); char *localhost_interface = "\0"; -#if defined(AWS_OS_APPLE) +# if defined(AWS_OS_APPLE) localhost_interface = "lo0"; -#else +# else localhost_interface = "lo"; -#endif +# endif interface_names_array[0] = aws_byte_cursor_from_c_str(localhost_interface); interface_names_array[1] = aws_byte_cursor_from_c_str(localhost_interface); @@ -212,11 +215,11 @@ TEST_CASE(multipart_upload_with_network_interface_names_mock_server) { aws_s3_meta_request_test_results_init(&out_results, allocator); ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &put_options, &out_results)); if (out_results.finished_error_code != 0) { -#if !defined(AWS_OS_APPLE) && !defined(AWS_OS_LINUX) +# if !defined(AWS_OS_APPLE) && !defined(AWS_OS_LINUX) if (out_results.finished_error_code == AWS_ERROR_PLATFORM_NOT_SUPPORTED) { return AWS_OP_SKIP; } -#endif +# endif ASSERT_TRUE(false, "aws_s3_tester_send_meta_request_with_options(() failed"); } aws_s3_meta_request_test_results_clean_up(&out_results); @@ -225,6 +228,7 @@ TEST_CASE(multipart_upload_with_network_interface_names_mock_server) { aws_mem_release(allocator, interface_names_array); return AWS_OP_SUCCESS; +#endif } TEST_CASE(multipart_upload_checksum_with_retry_mock_server) { diff --git a/tests/s3_s3express_client_test.c b/tests/s3_s3express_client_test.c index 7c2e5a017..299dc9ef9 100644 --- a/tests/s3_s3express_client_test.c +++ b/tests/s3_s3express_client_test.c @@ -648,3 +648,47 @@ TEST_CASE(s3express_client_get_object_create_session_error) { aws_s3_tester_clean_up(&tester); return AWS_OP_SUCCESS; } + +/** + * Test copy object within the same directory bucket. + */ +TEST_CASE(s3express_client_copy_object) { + (void)ctx; + + struct aws_byte_cursor source_key = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("pre-existing-10MB"); + struct aws_byte_cursor destination_key = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("copies/destination_10MB"); + struct aws_byte_cursor source_bucket = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("aws-c-s3-test-bucket--usw2-az1--x-s3"); + return aws_test_s3_copy_object_helper( + allocator, + source_bucket, + source_key, + g_test_s3express_bucket_usw2_az1_endpoint, + destination_key, + AWS_ERROR_SUCCESS, + AWS_HTTP_STATUS_CODE_200_OK, + MB_TO_BYTES(10), + true); +} + +/** + * Test multipart copy object within the same directory bucket. + */ +TEST_CASE(s3express_client_copy_object_multipart) { + (void)ctx; + + struct aws_byte_cursor source_key = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("pre-existing-2GB"); + struct aws_byte_cursor destination_key = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("copies/destination_2GB"); + struct aws_byte_cursor source_bucket = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("aws-c-s3-test-bucket--usw2-az1--x-s3"); + return aws_test_s3_copy_object_helper( + allocator, + source_bucket, + source_key, + g_test_s3express_bucket_usw2_az1_endpoint, + destination_key, + AWS_ERROR_SUCCESS, + AWS_HTTP_STATUS_CODE_200_OK, + GB_TO_BYTES(2), + true); +} diff --git a/tests/s3_tester.c b/tests/s3_tester.c index e58a396a3..d0283094e 100644 --- a/tests/s3_tester.c +++ b/tests/s3_tester.c @@ -2217,3 +2217,261 @@ struct aws_string *aws_s3_tester_create_file( return filepath_str; } + +static const struct aws_byte_cursor g_x_amz_copy_source_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"); + +static struct aws_http_message *s_copy_object_request_new( + struct aws_allocator *allocator, + struct aws_byte_cursor x_amz_source, + struct aws_byte_cursor endpoint, + struct aws_byte_cursor destination_key) { + + AWS_PRECONDITION(allocator); + + struct aws_http_message *message = aws_http_message_new_request(allocator); + + if (message == NULL) { + return NULL; + } + + /* the URI path is / followed by the key */ + char destination_path[1024]; + snprintf(destination_path, sizeof(destination_path), "/%.*s", (int)destination_key.len, destination_key.ptr); + struct aws_byte_cursor unencoded_destination_path = aws_byte_cursor_from_c_str(destination_path); + struct aws_byte_buf copy_destination_path_encoded; + aws_byte_buf_init(©_destination_path_encoded, allocator, 1024); + aws_byte_buf_append_encoding_uri_path(©_destination_path_encoded, &unencoded_destination_path); + if (aws_http_message_set_request_path(message, aws_byte_cursor_from_buf(©_destination_path_encoded))) { + goto error_clean_up_message; + } + + struct aws_http_header host_header = {.name = g_host_header_name, .value = endpoint}; + if (aws_http_message_add_header(message, host_header)) { + goto error_clean_up_message; + } + + struct aws_byte_buf copy_source_value_encoded; + aws_byte_buf_init(©_source_value_encoded, allocator, 1024); + aws_byte_buf_append_encoding_uri_path(©_source_value_encoded, &x_amz_source); + + struct aws_http_header copy_source_header = { + .name = g_x_amz_copy_source_name, + .value = aws_byte_cursor_from_buf(©_source_value_encoded), + }; + + if (aws_http_message_add_header(message, copy_source_header)) { + goto error_clean_up_message; + } + + if (aws_http_message_set_request_method(message, aws_http_method_put)) { + goto error_clean_up_message; + } + + aws_byte_buf_clean_up(©_source_value_encoded); + aws_byte_buf_clean_up(©_destination_path_encoded); + return message; + +error_clean_up_message: + + aws_byte_buf_clean_up(©_source_value_encoded); + aws_byte_buf_clean_up(©_destination_path_encoded); + if (message != NULL) { + aws_http_message_release(message); + message = NULL; + } + + return NULL; +} + +struct copy_object_test_data { + struct aws_mutex mutex; + struct aws_condition_variable c_var; + bool execution_completed; + bool headers_callback_was_invoked; + int meta_request_error_code; + int response_status_code; + uint64_t progress_callback_content_length; + uint64_t progress_callback_total_bytes_transferred; +}; + +static void s_copy_object_meta_request_finish( + struct aws_s3_meta_request *meta_request, + const struct aws_s3_meta_request_result *meta_request_result, + void *user_data) { + + (void)meta_request; + + struct copy_object_test_data *test_data = user_data; + + /* if error response body is available, dump it to test result to help investigation of failed tests */ + if (meta_request_result->error_response_body != NULL && meta_request_result->error_response_body->len > 0) { + AWS_LOGF_ERROR( + AWS_LS_S3_GENERAL, + "Response error body: %.*s", + (int)meta_request_result->error_response_body->len, + meta_request_result->error_response_body->buffer); + } + + aws_mutex_lock(&test_data->mutex); + test_data->meta_request_error_code = meta_request_result->error_code; + test_data->response_status_code = meta_request_result->response_status; + test_data->execution_completed = true; + aws_mutex_unlock(&test_data->mutex); + aws_condition_variable_notify_one(&test_data->c_var); +} + +static int s_copy_object_meta_request_headers_callback( + struct aws_s3_meta_request *meta_request, + const struct aws_http_headers *headers, + int response_status, + void *user_data) { + + (void)meta_request; + (void)headers; + (void)response_status; + + struct copy_object_test_data *test_data = user_data; + + aws_mutex_lock(&test_data->mutex); + test_data->headers_callback_was_invoked = true; + aws_mutex_unlock(&test_data->mutex); + + return AWS_OP_SUCCESS; +} + +static void s_copy_object_meta_request_progress_callback( + struct aws_s3_meta_request *meta_request, + const struct aws_s3_meta_request_progress *progress, + void *user_data) { + + (void)meta_request; + struct copy_object_test_data *test_data = user_data; + + aws_mutex_lock(&test_data->mutex); + test_data->progress_callback_content_length = progress->content_length; + test_data->progress_callback_total_bytes_transferred += progress->bytes_transferred; + aws_mutex_unlock(&test_data->mutex); +} + +static bool s_copy_test_completion_predicate(void *arg) { + struct copy_object_test_data *test_data = arg; + return test_data->execution_completed; +} + +int aws_test_s3_copy_object_from_x_amz_copy_source( + struct aws_allocator *allocator, + struct aws_byte_cursor x_amz_copy_source, + struct aws_byte_cursor destination_endpoint, + struct aws_byte_cursor destination_key, + int expected_error_code, + int expected_response_status, + uint64_t expected_size, + bool s3express) { + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_client_config client_config; + AWS_ZERO_STRUCT(client_config); + client_config.enable_s3express = s3express; + struct aws_byte_cursor region_cursor = g_test_s3_region; + client_config.region = region_cursor; + ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_SIGNING)); + + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + + /* creates a CopyObject request */ + struct aws_http_message *message = + s_copy_object_request_new(allocator, x_amz_copy_source, destination_endpoint, destination_key); + + struct copy_object_test_data test_data; + AWS_ZERO_STRUCT(test_data); + + struct aws_signing_config_aws s3express_signing_config = { + .algorithm = AWS_SIGNING_ALGORITHM_V4_S3EXPRESS, + .service = g_s3express_service_name, + }; + test_data.c_var = (struct aws_condition_variable)AWS_CONDITION_VARIABLE_INIT; + aws_mutex_init(&test_data.mutex); + + struct aws_s3_meta_request_options meta_request_options = { + .user_data = &test_data, + .body_callback = NULL, + .finish_callback = s_copy_object_meta_request_finish, + .headers_callback = s_copy_object_meta_request_headers_callback, + .progress_callback = s_copy_object_meta_request_progress_callback, + .message = message, + .shutdown_callback = NULL, + .signing_config = client_config.signing_config, + .type = AWS_S3_META_REQUEST_TYPE_COPY_OBJECT, + }; + + if (s3express) { + meta_request_options.signing_config = &s3express_signing_config; + } + + struct aws_s3_meta_request *meta_request = aws_s3_client_make_meta_request(client, &meta_request_options); + ASSERT_NOT_NULL(meta_request); + + /* wait completion of the meta request */ + aws_mutex_lock(&test_data.mutex); + aws_condition_variable_wait_pred(&test_data.c_var, &test_data.mutex, s_copy_test_completion_predicate, &test_data); + aws_mutex_unlock(&test_data.mutex); + + /* assert error_code and response_status_code */ + ASSERT_INT_EQUALS(expected_error_code, test_data.meta_request_error_code); + ASSERT_INT_EQUALS(expected_response_status, test_data.response_status_code); + + /* assert that progress_callback matches the expected size*/ + if (test_data.meta_request_error_code == AWS_ERROR_SUCCESS) { + ASSERT_UINT_EQUALS(expected_size, test_data.progress_callback_total_bytes_transferred); + ASSERT_UINT_EQUALS(expected_size, test_data.progress_callback_content_length); + } + + /* assert headers callback was invoked */ + ASSERT_TRUE(test_data.headers_callback_was_invoked); + + aws_s3_meta_request_release(meta_request); + aws_mutex_clean_up(&test_data.mutex); + aws_http_message_destroy(message); + client = aws_s3_client_release(client); + + aws_s3_tester_clean_up(&tester); + + return 0; +} + +int aws_test_s3_copy_object_helper( + struct aws_allocator *allocator, + struct aws_byte_cursor source_bucket, + struct aws_byte_cursor source_key, + struct aws_byte_cursor destination_endpoint, + struct aws_byte_cursor destination_key, + int expected_error_code, + int expected_response_status, + uint64_t expected_size, + bool s3_express) { + + char copy_source_value[1024]; + snprintf( + copy_source_value, + sizeof(copy_source_value), + "%.*s/%.*s", + (int)source_bucket.len, + source_bucket.ptr, + (int)source_key.len, + source_key.ptr); + + struct aws_byte_cursor x_amz_copy_source = aws_byte_cursor_from_c_str(copy_source_value); + + return aws_test_s3_copy_object_from_x_amz_copy_source( + allocator, + x_amz_copy_source, + destination_endpoint, + destination_key, + expected_error_code, + expected_response_status, + expected_size, + s3_express); +} diff --git a/tests/s3_tester.h b/tests/s3_tester.h index e7874351f..5c1c41e82 100644 --- a/tests/s3_tester.h +++ b/tests/s3_tester.h @@ -516,4 +516,31 @@ extern struct aws_byte_cursor g_test_s3express_bucket_usw2_az1_endpoint; * aws-c-s3-test-bucket--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com */ extern struct aws_byte_cursor g_test_s3express_bucket_use1_az4_endpoint; +/** + * Take the source with source_bucket and source_key, and the destination_endpoint to help testing copy object + */ +int aws_test_s3_copy_object_helper( + struct aws_allocator *allocator, + struct aws_byte_cursor source_bucket, + struct aws_byte_cursor source_key, + struct aws_byte_cursor destination_endpoint, + struct aws_byte_cursor destination_key, + int expected_error_code, + int expected_response_status, + uint64_t expected_size, + bool s3_express); + +/** + * Take the source with x_amz_copy_source, and the destination_endpoint to help testing copy object. + */ +int aws_test_s3_copy_object_from_x_amz_copy_source( + struct aws_allocator *allocator, + struct aws_byte_cursor x_amz_copy_source, + struct aws_byte_cursor destination_endpoint, + struct aws_byte_cursor destination_key, + int expected_error_code, + int expected_response_status, + uint64_t expected_size, + bool s3_express); + #endif /* AWS_S3_TESTER_H */ diff --git a/tests/test_helper/README.md b/tests/test_helper/README.md index 7124f190a..7a07ad947 100644 --- a/tests/test_helper/README.md +++ b/tests/test_helper/README.md @@ -46,10 +46,14 @@ python3 test_helper.py clean * Create directory bucket `--usw2-az1--x-s3` in us-west-2 * Upload files: + `pre-existing-10MB` 10MB file. + + with `--large_objects` enabled + - `pre-existing-2GB` * Create directory bucket `--use1-az4--x-s3` in us-east-1 * Upload files: + `pre-existing-10MB` 10MB file. + + with `--large_objects` enabled + - `pre-existing-2GB` ### `clean` action diff --git a/tests/test_helper/test_helper.py b/tests/test_helper/test_helper.py index c58c0cef7..62cdc54b0 100755 --- a/tests/test_helper/test_helper.py +++ b/tests/test_helper/test_helper.py @@ -16,9 +16,6 @@ s3_client_east1 = boto3.client('s3', region_name=REGION_EAST_1) -s3_control_client = boto3.client('s3control') - - MB = 1024*1024 GB = 1024*1024*1024 @@ -166,6 +163,10 @@ def create_bucket_with_lifecycle(availability_zone=None, client=s3_client): put_pre_existing_objects( 10*MB, 'pre-existing-10MB', bucket=bucket_name, client=client) + if args.large_objects: + put_pre_existing_objects( + 2*GB, 'pre-existing-2GB', bucket=bucket_name, client=client) + if availability_zone is None: put_pre_existing_objects( 10*MB, 'pre-existing-10MB-aes256-c', sse='aes256-c', bucket=bucket_name) @@ -186,8 +187,6 @@ def create_bucket_with_lifecycle(availability_zone=None, client=s3_client): 256*MB, 'pre-existing-256MB', bucket=bucket_name) put_pre_existing_objects( 256*MB, 'pre-existing-256MB-@', bucket=bucket_name) - put_pre_existing_objects( - 2*GB, 'pre-existing-2GB', bucket=bucket_name) put_pre_existing_objects( 2*GB, 'pre-existing-2GB-@', bucket=bucket_name) else: