Skip to content

Commit

Permalink
Merge branch 'main' into support-header-checksum
Browse files Browse the repository at this point in the history
  • Loading branch information
@TingDaoK
TingDaoK authored Oct 16, 2024
2 parents dce676c + 45eee8d commit 6922c0c
Show file tree
Hide file tree
Showing 14 changed files with 560 additions and 285 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,6 @@ cmake-build*
# js package locks irrelevant to the overall package's security
benchmarks/benchmarks-stack/benchmarks-stack/package-lock.json
benchmarks/dashboard-stack/package-lock.json

# virtual environment
.venv/
8 changes: 8 additions & 0 deletions include/aws/s3/private/s3_meta_request_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,14 @@ void aws_s3_meta_request_init_signing_date_time_default(
struct aws_s3_meta_request *meta_request,
struct aws_date_time *date_time);

AWS_S3_API
void aws_s3_meta_request_sign_request_default_impl(
struct aws_s3_meta_request *meta_request,
struct aws_s3_request *request,
aws_signing_complete_fn *on_signing_complete,
void *user_data,
bool disable_s3_express_signing);

AWS_S3_API
void aws_s3_meta_request_sign_request_default(
struct aws_s3_meta_request *meta_request,
Expand Down
1 change: 1 addition & 0 deletions include/aws/s3/s3_client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ enum aws_s3_meta_request_type {
* - only {bucket}/{key} format is supported for source and passing arn as
* source will not work
* - source bucket is assumed to be in the same region as dest
* - source bucket and dest bucket must both be either directory buckets or regular buckets.
*/
AWS_S3_META_REQUEST_TYPE_COPY_OBJECT,

Expand Down
88 changes: 55 additions & 33 deletions source/s3_client.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ struct aws_s3_client *aws_s3_client_new(
client->buffer_pool = aws_s3_buffer_pool_new(allocator, part_size, mem_limit);

if (client->buffer_pool == NULL) {
goto on_early_fail;
goto on_error;
}

struct aws_s3_buffer_pool_usage_stats pool_usage = aws_s3_buffer_pool_get_usage(client->buffer_pool);
Expand All @@ -357,15 +357,15 @@ struct aws_s3_client *aws_s3_client_new(
"Cannot create client from client_config; configured max part size should not exceed memory limit."
"size.");
aws_raise_error(AWS_ERROR_S3_INVALID_MEMORY_LIMIT_CONFIG);
goto on_early_fail;
goto on_error;
}

client->vtable = &s_s3_client_default_vtable;

aws_ref_count_init(&client->ref_count, client, (aws_simple_completion_callback *)s_s3_client_start_destroy);

if (aws_mutex_init(&client->synced_data.lock) != AWS_OP_SUCCESS) {
goto on_early_fail;
goto on_error;
}

aws_linked_list_init(&client->synced_data.pending_meta_request_work);
Expand Down Expand Up @@ -488,6 +488,44 @@ struct aws_s3_client *aws_s3_client_new(
}
}

client->num_network_interface_names = client_config->num_network_interface_names;
if (client_config->num_network_interface_names > 0) {
AWS_LOGF_DEBUG(
AWS_LS_S3_CLIENT,
"id=%p Client received network interface names array with length %zu.",
(void *)client,
client->num_network_interface_names);
aws_array_list_init_dynamic(
&client->network_interface_names,
client->allocator,
client_config->num_network_interface_names,
sizeof(struct aws_string *));
client->network_interface_names_cursor_array = aws_mem_calloc(
client->allocator, client_config->num_network_interface_names, sizeof(struct aws_byte_cursor));
for (size_t i = 0; i < client_config->num_network_interface_names; i++) {
struct aws_byte_cursor interface_name = client_config->network_interface_names_array[i];
struct aws_string *interface_name_str = aws_string_new_from_cursor(client->allocator, &interface_name);
aws_array_list_push_back(&client->network_interface_names, &interface_name_str);
if (aws_is_network_interface_name_valid(aws_string_c_str(interface_name_str)) == false) {
AWS_LOGF_ERROR(
AWS_LS_S3_CLIENT,
"id=%p network_interface_names_array[%zu]=" PRInSTR " is not valid.",
(void *)client,
i,
AWS_BYTE_CURSOR_PRI(interface_name));
aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
goto on_error;
}
client->network_interface_names_cursor_array[i] = aws_byte_cursor_from_string(interface_name_str);
AWS_LOGF_DEBUG(
AWS_LS_S3_CLIENT,
"id=%p network_interface_names_array[%zu]=" PRInSTR "",
(void *)client,
i,
AWS_BYTE_CURSOR_PRI(client->network_interface_names_cursor_array[i]));
}
}

/* Set up body streaming ELG */
{
uint16_t num_event_loops =
Expand Down Expand Up @@ -579,34 +617,6 @@ struct aws_s3_client *aws_s3_client_new(
*((bool *)&client->enable_read_backpressure) = client_config->enable_read_backpressure;
*((size_t *)&client->initial_read_window) = client_config->initial_read_window;

client->num_network_interface_names = client_config->num_network_interface_names;
if (client_config->num_network_interface_names > 0) {
AWS_LOGF_DEBUG(
AWS_LS_S3_CLIENT,
"id=%p Client received network interface names array with length %zu.",
(void *)client,
client->num_network_interface_names);
aws_array_list_init_dynamic(
&client->network_interface_names,
client->allocator,
client_config->num_network_interface_names,
sizeof(struct aws_string *));
client->network_interface_names_cursor_array = aws_mem_calloc(
client->allocator, client_config->num_network_interface_names, sizeof(struct aws_byte_cursor));
for (size_t i = 0; i < client_config->num_network_interface_names; i++) {
struct aws_byte_cursor interface_name = client_config->network_interface_names_array[i];
struct aws_string *interface_name_str = aws_string_new_from_cursor(client->allocator, &interface_name);
aws_array_list_push_back(&client->network_interface_names, &interface_name_str);
client->network_interface_names_cursor_array[i] = aws_byte_cursor_from_string(interface_name_str);
AWS_LOGF_DEBUG(
AWS_LS_S3_CLIENT,
"id=%p network_interface_names_array[%zu]=" PRInSTR "",
(void *)client,
i,
AWS_BYTE_CURSOR_PRI(client->network_interface_names_cursor_array[i]));
}
}

return client;

on_error:
Expand All @@ -628,10 +638,22 @@ struct aws_s3_client *aws_s3_client_new(
aws_mem_release(client->allocator, client->proxy_ev_settings);
aws_mem_release(client->allocator, client->tcp_keep_alive_options);

aws_event_loop_group_release(client->client_bootstrap->event_loop_group);
if (client->client_bootstrap != NULL) {
aws_event_loop_group_release(client->client_bootstrap->event_loop_group);
}
aws_client_bootstrap_release(client->client_bootstrap);
aws_mutex_clean_up(&client->synced_data.lock);
on_early_fail:

aws_mem_release(client->allocator, client->network_interface_names_cursor_array);
for (size_t i = 0; i < aws_array_list_length(&client->network_interface_names); i++) {
struct aws_string *interface_name = NULL;
aws_array_list_get_at(&client->network_interface_names, &interface_name, i);
aws_string_destroy(interface_name);
}

aws_array_list_clean_up(&client->network_interface_names);
aws_s3_buffer_pool_destroy(client->buffer_pool);

aws_mem_release(client->allocator, client);
return NULL;
}
Expand Down
26 changes: 25 additions & 1 deletion source/s3_copy_object.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,18 @@ static void s_s3_copy_object_request_finished(
struct aws_s3_request *request,
int error_code);

static void s_s3_copy_object_sign_request(
struct aws_s3_meta_request *meta_request,
struct aws_s3_request *request,
aws_signing_complete_fn *on_signing_complete,
void *user_data);

static struct aws_s3_meta_request_vtable s_s3_copy_object_vtable = {
.update = s_s3_copy_object_update,
.send_request_finish = aws_s3_meta_request_send_request_finish_default,
.prepare_request = s_s3_copy_object_prepare_request,
.init_signing_date_time = aws_s3_meta_request_init_signing_date_time_default,
.sign_request = aws_s3_meta_request_sign_request_default,
.sign_request = s_s3_copy_object_sign_request,
.finished_request = s_s3_copy_object_request_finished,
.destroy = s_s3_meta_request_copy_object_destroy,
.finish = aws_s3_meta_request_finish_default,
Expand Down Expand Up @@ -796,3 +802,21 @@ static void s_s3_copy_object_request_finished(

aws_s3_meta_request_unlock_synced_data(meta_request);
}

static void s_s3_copy_object_sign_request(
struct aws_s3_meta_request *meta_request,
struct aws_s3_request *request,
aws_signing_complete_fn *on_signing_complete,
void *user_data) {

/**
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
* For CopyObject and UploadPartCopy, the request has to be signed with IAM credentials for directory buckets.
* Disable S3 express signing for those types.
*/
bool disable_s3_express_signing = request->request_tag == AWS_S3_COPY_OBJECT_REQUEST_TAG_BYPASS ||
request->request_tag == AWS_S3_COPY_OBJECT_REQUEST_TAG_MULTIPART_COPY;
aws_s3_meta_request_sign_request_default_impl(
meta_request, request, on_signing_complete, user_data, disable_s3_express_signing);
}
20 changes: 16 additions & 4 deletions source/s3_meta_request.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -900,12 +900,12 @@ static int s_meta_request_resolve_signing_config(
return AWS_OP_SUCCESS;
}

/* Handles signing a message for the caller. */
void aws_s3_meta_request_sign_request_default(
void aws_s3_meta_request_sign_request_default_impl(
struct aws_s3_meta_request *meta_request,
struct aws_s3_request *request,
aws_signing_complete_fn *on_signing_complete,
void *user_data) {
void *user_data,
bool disable_s3_express_signing) {
AWS_PRECONDITION(meta_request);
AWS_PRECONDITION(request);
AWS_PRECONDITION(on_signing_complete);
Expand Down Expand Up @@ -947,7 +947,7 @@ void aws_s3_meta_request_sign_request_default(
return;
}

if (signing_config.algorithm == AWS_SIGNING_ALGORITHM_V4_S3EXPRESS) {
if (signing_config.algorithm == AWS_SIGNING_ALGORITHM_V4_S3EXPRESS && !disable_s3_express_signing) {
/* Fetch credentials from S3 Express provider. */
struct aws_get_s3express_credentials_user_data *context =
aws_mem_calloc(meta_request->allocator, 1, sizeof(struct aws_get_s3express_credentials_user_data));
Expand Down Expand Up @@ -998,6 +998,9 @@ void aws_s3_meta_request_sign_request_default(
}
} else {
/* Regular signing. */
if (disable_s3_express_signing) {
signing_config.algorithm = AWS_SIGNING_ALGORITHM_V4;
}
s_s3_meta_request_init_signing_date_time(meta_request, &signing_config.date);
if (aws_sign_request_aws(
meta_request->allocator,
Expand All @@ -1015,6 +1018,15 @@ void aws_s3_meta_request_sign_request_default(
}
}

/* Handles signing a message for the caller. */
void aws_s3_meta_request_sign_request_default(
struct aws_s3_meta_request *meta_request,
struct aws_s3_request *request,
aws_signing_complete_fn *on_signing_complete,
void *user_data) {
aws_s3_meta_request_sign_request_default_impl(meta_request, request, on_signing_complete, user_data, false);
}

/* Handle the signing result */
static void s_s3_meta_request_request_on_signed(
struct aws_signing_result *signing_result,
Expand Down
4 changes: 4 additions & 0 deletions tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ add_test_case(test_s3_abort_multipart_upload_message_new)

add_net_test_case(test_s3_client_create_destroy)
add_net_test_case(test_s3_client_create_error)
add_net_test_case(test_s3_client_create_error_with_invalid_memory_limit_config)
add_net_test_case(test_s3_client_create_error_with_invalid_network_interface)
add_net_test_case(test_s3_client_monitoring_options_override)
add_net_test_case(test_s3_client_proxy_ev_settings_override)
add_net_test_case(test_s3_client_tcp_keep_alive_options_override)
Expand Down Expand Up @@ -352,6 +354,8 @@ add_net_test_case(s3express_client_put_object_long_running_session_refresh)
add_net_test_case(s3express_client_get_object)
add_net_test_case(s3express_client_get_object_multiple)
add_net_test_case(s3express_client_get_object_create_session_error)
add_net_test_case(s3express_client_copy_object)
add_net_test_case(s3express_client_copy_object_multipart)

add_net_test_case(meta_request_auto_ranged_get_new_error_handling)
add_net_test_case(meta_request_auto_ranged_put_new_error_handling)
Expand Down
Loading

0 comments on commit 6922c0c

Please sign in to comment.