Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support header checksum #454

Merged
merged 8 commits into from
Oct 17, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions include/aws/s3/s3_client.h
Original file line number Diff line number Diff line change
Expand Up @@ -558,19 +558,19 @@ struct aws_s3_checksum_config {
/**
* The location of client added checksum header.
*
* If AWS_SCL_NONE. No request payload checksum will be add and calculated.
* If AWS_SCL_NONE. No request payload checksum will be calculated or added.
*
* If AWS_SCL_HEADER, the checksum will be calculated by client and added related header to the request sent.
* If AWS_SCL_HEADER, the client will calculate the checksum and add it to the headers.
*
* If AWS_SCL_TRAILER, the payload will be aws_chunked encoded, The checksum will be calculate while reading the
* payload by client. Related header will be added to the trailer part of the encoded payload. Note the payload of
* the original request cannot be aws-chunked encoded already. Otherwise, error will be raised.
* If AWS_SCL_TRAILER, the payload will be aws_chunked encoded, The client will calculate the checksum and add it to
* the trailer. Note the payload of the original request cannot be aws-chunked encoded already, this will cause an
* error.
*/
enum aws_s3_checksum_location location;

/**
* The checksum algorithm used.
* Must be set if location is not AWS_SCL_NONE. Must be AWS_SCA_NONE if location is AWS_SCL_NONE.
* Must be set if location is not AWS_SCL_NONE.
*/
enum aws_s3_checksum_algorithm checksum_algorithm;

Expand Down
6 changes: 0 additions & 6 deletions source/s3_client.c
Original file line number Diff line number Diff line change
Expand Up @@ -941,12 +941,6 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
}
}

if (options->checksum_config->location == AWS_SCL_HEADER) {
/* TODO: support calculate checksum to add to header */
aws_raise_error(AWS_ERROR_UNSUPPORTED_OPERATION);
return NULL;
}

if (options->checksum_config->location != AWS_SCL_NONE &&
options->checksum_config->checksum_algorithm == AWS_SCA_NONE) {
AWS_LOGF_ERROR(
Expand Down
117 changes: 96 additions & 21 deletions source/s3_request_messages.c
Original file line number Diff line number Diff line change
Expand Up @@ -730,6 +730,91 @@ struct aws_http_message *aws_s3_abort_multipart_upload_message_new(
return NULL;
}

/**
* Calculate the in memory checksum based on the checksum config. Initialize and set the out_checksum to the encoded
* checksum result
*/
static int s_calculate_in_memory_checksum_helper(
struct aws_allocator *allocator,
struct aws_byte_cursor data,
const struct checksum_config *checksum_config,
struct aws_byte_buf *out_checksum) {
AWS_ASSERT(checksum_config->checksum_algorithm != AWS_SCA_NONE);
AWS_ASSERT(out_checksum != NULL);

int ret_code = AWS_OP_ERR;
size_t digest_size = aws_get_digest_size_from_algorithm(checksum_config->checksum_algorithm);
size_t encoded_checksum_len = 0;
if (aws_base64_compute_encoded_len(digest_size, &encoded_checksum_len)) {
return ret_code;
}

aws_byte_buf_init(out_checksum, allocator, encoded_checksum_len);

struct aws_byte_buf raw_checksum;
aws_byte_buf_init(&raw_checksum, allocator, digest_size);

if (aws_checksum_compute(allocator, checksum_config->checksum_algorithm, &data, &raw_checksum, 0 /*truncate_to*/)) {
goto done;
}
struct aws_byte_cursor raw_checksum_cursor = aws_byte_cursor_from_buf(&raw_checksum);
if (aws_base64_encode(&raw_checksum_cursor, out_checksum)) {
goto done;
}

ret_code = AWS_OP_SUCCESS;
done:
aws_byte_buf_clean_up(&raw_checksum);
return ret_code;
TingDaoK marked this conversation as resolved.
Show resolved Hide resolved
}

/**
* Calculate the in memory checksum based on the checksum config.
* If out_checksum set, initialize and set it to the encoded checksum result.
* Set the corresponding header in out_message to the encoded checksum result.
*/
static int s_calculate_and_add_checksum_to_header_helper(
struct aws_allocator *allocator,
struct aws_byte_cursor data,
const struct checksum_config *checksum_config,
struct aws_http_message *out_message,
struct aws_byte_buf *out_checksum) {
AWS_ASSERT(checksum_config->checksum_algorithm != AWS_SCA_NONE);
AWS_ASSERT(out_message != NULL);
int ret_code = AWS_OP_ERR;

struct aws_byte_buf *local_encoded_checksum = NULL;
if (out_checksum == NULL) {
local_encoded_checksum = aws_mem_calloc(allocator, 1, sizeof(struct aws_byte_buf));
} else {
local_encoded_checksum = out_checksum;
}

if (s_calculate_in_memory_checksum_helper(allocator, data, checksum_config, local_encoded_checksum)) {
goto done;
}

/* Add the encoded checksum to header. */
const struct aws_byte_cursor *header_name =
aws_get_http_header_name_from_algorithm(checksum_config->checksum_algorithm);
struct aws_byte_cursor encoded_checksum_val = aws_byte_cursor_from_buf(local_encoded_checksum);
struct aws_http_headers *headers = aws_http_message_get_headers(out_message);
if (aws_http_headers_set(headers, *header_name, encoded_checksum_val)) {
goto done;
}

ret_code = AWS_OP_SUCCESS;
done:
if (out_checksum == NULL) {
/* In case out_checksum is not set. In this case, we need to clean up the local_encoded_checksum.
* Otherwise, the caller will own the out_checksum.
*/
aws_byte_buf_clean_up(local_encoded_checksum);
aws_mem_release(allocator, local_encoded_checksum);
}
return ret_code;
TingDaoK marked this conversation as resolved.
Show resolved Hide resolved
}

/* Assign a buffer to an HTTP message, creating a stream and setting the content-length header */
struct aws_input_stream *aws_s3_message_util_assign_body(
struct aws_allocator *allocator,
Expand Down Expand Up @@ -812,25 +897,19 @@ struct aws_input_stream *aws_s3_message_util_assign_body(
}
aws_input_stream_release(input_stream);
input_stream = chunk_stream;
} else if (
checksum_config->checksum_algorithm != AWS_SCA_NONE && checksum_config->location == AWS_SCL_NONE &&
out_checksum != NULL) {
/* The checksum won't be uploaded, but we still need it for the upload review callback */
size_t checksum_len = aws_get_digest_size_from_algorithm(checksum_config->checksum_algorithm);
size_t encoded_checksum_len = 0;
if (aws_base64_compute_encoded_len(checksum_len, &encoded_checksum_len)) {
} else if (checksum_config->location == AWS_SCL_HEADER) {
/* Calculate the checksum directly from memory and add it to the header. */
if (s_calculate_and_add_checksum_to_header_helper(
allocator, buffer_byte_cursor, checksum_config, out_message, out_checksum)) {
goto error_clean_up;
}
if (aws_byte_buf_init(out_checksum, allocator, encoded_checksum_len)) {
goto error_clean_up;
}
struct aws_input_stream *checksum_stream =
aws_checksum_stream_new(allocator, input_stream, checksum_config->checksum_algorithm, out_checksum);
if (!checksum_stream) {

} else if (checksum_config->checksum_algorithm != AWS_SCA_NONE && out_checksum != NULL) {
/* In case checksums still wanted, and we can calculate it directly from the buffer in memory to
* out_checksum */
if (s_calculate_in_memory_checksum_helper(allocator, buffer_byte_cursor, checksum_config, out_checksum)) {
goto error_clean_up;
}
aws_input_stream_release(input_stream);
input_stream = checksum_stream;
}
}
int64_t stream_length = 0;
Expand Down Expand Up @@ -892,9 +971,7 @@ int aws_s3_message_util_add_content_md5_header(
return AWS_OP_ERR;
}
struct aws_byte_buf base64_output_buf;
if (aws_byte_buf_init(&base64_output_buf, allocator, base64_output_size)) {
return AWS_OP_ERR;
}
aws_byte_buf_init(&base64_output_buf, allocator, base64_output_size);
if (aws_base64_encode(&base64_input, &base64_output_buf)) {
goto error_clean_up;
}
Expand Down Expand Up @@ -1056,9 +1133,7 @@ int aws_s3_message_util_set_multipart_request_path(
return AWS_OP_ERR;
}

if (aws_byte_buf_init(&request_path_buf, allocator, request_path.len)) {
return AWS_OP_ERR;
}
aws_byte_buf_init(&request_path_buf, allocator, request_path.len);

if (aws_byte_buf_append_dynamic(&request_path_buf, &request_path)) {
goto error_clean_up;
Expand Down
4 changes: 4 additions & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ add_net_test_case(test_s3_put_object_async_no_content_length_2parts)
add_net_test_case(test_s3_put_object_async_fail_reading)
add_net_test_case(test_s3_many_async_uploads_without_data)
add_net_test_case(test_s3_download_empty_file_with_checksum)
add_net_test_case(test_s3_download_empty_file_with_checksum_header)
add_net_test_case(test_s3_download_single_part_file_with_checksum)
add_net_test_case(test_s3_download_multipart_file_with_checksum)
add_net_test_case(test_s3_asyncwrite_empty_file)
Expand Down Expand Up @@ -174,6 +175,9 @@ add_net_test_case(test_s3_round_trip_default_get_fc)
add_net_test_case(test_s3_round_trip_mpu_multipart_get_fc)
add_net_test_case(test_s3_round_trip_mpu_multipart_get_with_list_algorithm_fc)
add_net_test_case(test_s3_round_trip_mpu_default_get_fc)
add_net_test_case(test_s3_round_trip_default_get_fc_header)
add_net_test_case(test_s3_round_trip_multipart_get_fc_header)
add_net_test_case(test_s3_round_trip_mpu_multipart_get_fc_header)
add_net_test_case(test_s3_round_trip_with_filepath)
add_net_test_case(test_s3_round_trip_mpu_with_filepath)
add_net_test_case(test_s3_round_trip_with_filepath_no_content_length)
Expand Down
60 changes: 52 additions & 8 deletions tests/s3_data_plane_tests.c
Original file line number Diff line number Diff line change
Expand Up @@ -3775,8 +3775,7 @@ void s_s3_test_no_validate_checksum(
}

/* TODO: maybe refactor the fc -> flexible checksum tests to be less copy/paste */
AWS_TEST_CASE(test_s3_round_trip_default_get_fc, s_test_s3_round_trip_default_get_fc)
static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator, void *ctx) {
static int s_test_s3_round_trip_default_get_fc_helper(struct aws_allocator *allocator, void *ctx, bool via_header) {
(void)ctx;

struct aws_s3_tester tester;
Expand Down Expand Up @@ -3810,6 +3809,7 @@ static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator,
.client = client,
.checksum_algorithm = algorithm,
.validate_get_response_checksum = false,
.checksum_via_header = via_header,
.put_options =
{
.object_size_mb = 1,
Expand Down Expand Up @@ -3846,8 +3846,17 @@ static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator,
return 0;
}

AWS_TEST_CASE(test_s3_round_trip_multipart_get_fc, s_test_s3_round_trip_multipart_get_fc)
static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator, void *ctx) {
AWS_TEST_CASE(test_s3_round_trip_default_get_fc, s_test_s3_round_trip_default_get_fc)
static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator, void *ctx) {
return s_test_s3_round_trip_default_get_fc_helper(allocator, ctx, false);
}

AWS_TEST_CASE(test_s3_round_trip_default_get_fc_header, s_test_s3_round_trip_default_get_fc_header)
static int s_test_s3_round_trip_default_get_fc_header(struct aws_allocator *allocator, void *ctx) {
return s_test_s3_round_trip_default_get_fc_helper(allocator, ctx, true);
}

static int s_test_s3_round_trip_multipart_get_fc_helper(struct aws_allocator *allocator, void *ctx, bool via_header) {
(void)ctx;

struct aws_s3_tester tester;
Expand All @@ -3873,6 +3882,7 @@ static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator
.client = client,
.checksum_algorithm = AWS_SCA_CRC32,
.validate_get_response_checksum = false,
.checksum_via_header = via_header,
.put_options =
{
.object_size_mb = 1,
Expand Down Expand Up @@ -3908,10 +3918,21 @@ static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator
return 0;
}

AWS_TEST_CASE(test_s3_round_trip_multipart_get_fc, s_test_s3_round_trip_multipart_get_fc)
static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator, void *ctx) {
return s_test_s3_round_trip_multipart_get_fc_helper(allocator, ctx, false);
}
AWS_TEST_CASE(test_s3_round_trip_multipart_get_fc_header, s_test_s3_round_trip_multipart_get_fc_header)
static int s_test_s3_round_trip_multipart_get_fc_header(struct aws_allocator *allocator, void *ctx) {
return s_test_s3_round_trip_multipart_get_fc_helper(allocator, ctx, true);
}

/* Test the multipart uploaded object was downloaded with same part size, which will download the object matches all the
* parts and validate the parts checksum. */
AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc, s_test_s3_round_trip_mpu_multipart_get_fc)
static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *allocator, void *ctx) {
static int s_test_s3_round_trip_mpu_multipart_get_fc_helper(
struct aws_allocator *allocator,
void *ctx,
bool via_header) {
(void)ctx;

struct aws_s3_tester tester;
Expand All @@ -3937,6 +3958,7 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *alloc
.client = client,
.checksum_algorithm = AWS_SCA_CRC32,
.validate_get_response_checksum = false,
.checksum_via_header = via_header,
.put_options =
{
.object_size_mb = 10,
Expand Down Expand Up @@ -3971,8 +3993,20 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *alloc
return 0;
}

AWS_TEST_CASE(test_s3_download_empty_file_with_checksum, s_test_s3_download_empty_file_with_checksum)
static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *allocator, void *ctx) {
AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc, s_test_s3_round_trip_mpu_multipart_get_fc)
static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *allocator, void *ctx) {
return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, false);
}

AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc_header, s_test_s3_round_trip_mpu_multipart_get_fc_header)
static int s_test_s3_round_trip_mpu_multipart_get_fc_header(struct aws_allocator *allocator, void *ctx) {
return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, true);
}

static int s_test_s3_download_empty_file_with_checksum_helper(
struct aws_allocator *allocator,
void *ctx,
bool via_header) {
(void)ctx;

/* Upload the file */
Expand All @@ -3998,6 +4032,7 @@ static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *all
.meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
.client = client,
.checksum_algorithm = AWS_SCA_CRC32,
.checksum_via_header = via_header,
.put_options =
{
.object_size_mb = 0,
Expand Down Expand Up @@ -4037,6 +4072,15 @@ static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *all
return 0;
}

AWS_TEST_CASE(test_s3_download_empty_file_with_checksum, s_test_s3_download_empty_file_with_checksum)
static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *allocator, void *ctx) {
return s_test_s3_download_empty_file_with_checksum_helper(allocator, ctx, false);
}
AWS_TEST_CASE(test_s3_download_empty_file_with_checksum_header, s_test_s3_download_empty_file_with_checksum_header)
static int s_test_s3_download_empty_file_with_checksum_header(struct aws_allocator *allocator, void *ctx) {
return s_test_s3_download_empty_file_with_checksum_helper(allocator, ctx, true);
}

AWS_TEST_CASE(test_s3_download_single_part_file_with_checksum, s_test_s3_download_single_part_file_with_checksum)
static int s_test_s3_download_single_part_file_with_checksum(struct aws_allocator *allocator, void *ctx) {
(void)ctx;
Expand Down
4 changes: 3 additions & 1 deletion tests/s3_tester.c
Original file line number Diff line number Diff line change
Expand Up @@ -1452,9 +1452,11 @@ int aws_s3_tester_send_meta_request_with_options(
struct aws_s3_checksum_config checksum_config = {
.checksum_algorithm = options->checksum_algorithm,
.validate_response_checksum = options->validate_get_response_checksum,
.location = disable_trailing_checksum ? AWS_SCL_NONE : AWS_SCL_TRAILER,
.validate_checksum_algorithms = options->validate_checksum_algorithms,
};
if (!disable_trailing_checksum) {
checksum_config.location = options->checksum_via_header ? AWS_SCL_HEADER : AWS_SCL_TRAILER;
}

struct aws_s3_meta_request_options meta_request_options = {
.type = options->meta_request_type,
Expand Down
1 change: 1 addition & 0 deletions tests/s3_tester.h
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ struct aws_s3_tester_meta_request_options {
struct aws_array_list *validate_checksum_algorithms;
enum aws_s3_checksum_algorithm expected_validate_checksum_alg;
bool disable_put_trailing_checksum;
bool checksum_via_header;

/* override client signing config */
struct aws_signing_config_aws *signing_config;
Expand Down