From 4c5cd918d850e24c94a29f8649b8c07ed874dfa2 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Thu, 27 Feb 2025 20:21:07 -0300 Subject: [PATCH 1/2] test: crc64nvme checksum --- tests/util/util_head_object.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/util/util_head_object.sh b/tests/util/util_head_object.sh index 3814f6ac..f7381120 100644 --- a/tests/util/util_head_object.sh +++ b/tests/util/util_head_object.sh @@ -166,3 +166,13 @@ check_checksum_rest_crc32() { fi return 0 } + +crc64_file() { + source ./env/bin/activate + python3 -m pip install awscrt + if ! checksum=$(python3 -c "import sys;from awscrt import checksums;crc64nvme_value = checksums.crc64nvme(sys.argv[1]);print(crc64nvme_value)" "$1"); then + return 1 + fi + log 5 "checksum: $checksum" + deactivate +} From 485f2abbaa387b851079423570aec842c1fa07ac Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Sat, 1 Mar 2025 22:58:18 -0300 Subject: [PATCH 2/2] test: crc32c checksum --- tests/env.sh | 74 ++++++++------- tests/rest_scripts/calculate_crc64nvme.sh | 10 +- tests/rest_scripts/put_object.sh | 21 ++++- tests/test_rest.sh | 5 +- tests/test_rest_checksum.sh | 78 +++++++++++---- tests/test_s3api_object.sh | 9 -- tests/util/util_head_object.sh | 10 -- tests/util/util_multipart.sh | 48 ++++++---- tests/util/util_object.sh | 110 +++++++++++++++------- 9 files changed, 231 insertions(+), 134 deletions(-) diff --git a/tests/env.sh b/tests/env.sh index 17cad190..d32ba7a0 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -58,29 +58,7 @@ source_config_file() { fi } -check_universal_vars() { - if [[ $BYPASS_ENV_FILE != "true" ]]; then - source_config_file - fi - if [ -n "$COMMAND_LOG" ]; then - init_command_log - fi - - if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then - # shellcheck source=./tests/.secrets - source "$SECRETS_FILE" - else - log 3 "Warning: no secrets file found" - fi - - if [[ -n "$LOG_LEVEL" ]]; then - if [[ $LOG_LEVEL -lt 2 ]]; then - log 1 "log level must be 2 or greater" - exit 1 - fi - export LOG_LEVEL_INT=$LOG_LEVEL - fi - +check_aws_vars() { if [ -z "$AWS_ACCESS_KEY_ID" ]; then log 1 "AWS_ACCESS_KEY_ID missing" exit 1 @@ -103,10 +81,14 @@ check_universal_vars() { exit 1 fi fi - if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then - fail "RUN_VERSITYGW must be 'true' or 'false'" + # exporting these since they're needed for subshells + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL + if [ -n "$AWS_CANONICAL_ID" ]; then + export AWS_CANONICAL_ID fi +} +check_bucket_vars() { if [ -z "$BUCKET_ONE_NAME" ]; then log 1 "BUCKET_ONE_NAME missing" exit 1 @@ -129,24 +111,50 @@ check_universal_vars() { fi if [ "$RECREATE_BUCKETS" == "false" ] && [ "$DELETE_BUCKETS_AFTER_TEST" == "true" ]; then log 1 "cannot set DELETE_BUCKETS_AFTER_TEST to 'true' if RECREATE_BUCKETS is 'false'" - return 1 + exit 1 + fi +} + +check_universal_vars() { + if [[ $BYPASS_ENV_FILE != "true" ]]; then + source_config_file + fi + if [ -n "$COMMAND_LOG" ]; then + init_command_log + fi + if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then + # shellcheck source=./tests/.secrets + source "$SECRETS_FILE" + else + log 3 "Warning: no secrets file found" + fi + if [[ -n "$LOG_LEVEL" ]]; then + if [[ $LOG_LEVEL -lt 2 ]]; then + log 1 "log level must be 2 or greater" + exit 1 + fi + export LOG_LEVEL_INT=$LOG_LEVEL + fi + + check_aws_vars + + if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then + log 1 "RUN_VERSITYGW must be 'true' or 'false'" + exit 1 fi + + check_bucket_vars + if [ -z "$TEST_FILE_FOLDER" ]; then log 1 "TEST_FILE_FOLDER missing" exit 1 fi if [ ! -d "$TEST_FILE_FOLDER" ]; then if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then - log 2 "error creating test folder: $error" + log 1 "error creating test folder: $error" exit 1 fi fi - # exporting these since they're needed for subshells - export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL - if [ -n "$AWS_CANONICAL_ID" ]; then - log 5 "canonical ID: $AWS_CANONICAL_ID" - export AWS_CANONICAL_ID - fi } delete_command_log() { diff --git a/tests/rest_scripts/calculate_crc64nvme.sh b/tests/rest_scripts/calculate_crc64nvme.sh index d7e90501..6409c9c3 100755 --- a/tests/rest_scripts/calculate_crc64nvme.sh +++ b/tests/rest_scripts/calculate_crc64nvme.sh @@ -25,7 +25,7 @@ import sys from awscrt import checksums with open(sys.argv[1], 'rb') as f: - print(checksums.crc64nvme(f.read()))" "$DATA_FILE" 2>&1); then + print(checksums.${CHECKSUM_TYPE}(f.read()))" "$DATA_FILE" 2>&1); then log_rest 2 "error calculating checksum: $checksum_decimal" exit 1 fi @@ -34,5 +34,11 @@ if ! deactivate 1>/dev/null; then log_rest 2 "error deactivating virtual environment" exit 1 fi -checksum_hash=$(printf "%016x" "$checksum_decimal" | xxd -r -p | base64) +if [ "$CHECKSUM_TYPE" == "crc64nvme" ]; then + hex_format="%016x" +else + hex_format="%08x" +fi +# shellcheck disable=SC2059 +checksum_hash=$(printf "$hex_format" "$checksum_decimal" | xxd -r -p | base64) echo "$checksum_hash" \ No newline at end of file diff --git a/tests/rest_scripts/put_object.sh b/tests/rest_scripts/put_object.sh index 21336de8..dc564658 100755 --- a/tests/rest_scripts/put_object.sh +++ b/tests/rest_scripts/put_object.sh @@ -46,19 +46,30 @@ if [ "$checksum_type" == "sha256" ]; then checksum_hash="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)" fi cr_data+=("x-amz-checksum-sha256:$checksum_hash") +elif [ "$checksum_type" == "sha1" ]; then + if [ -z "$checksum_hash" ]; then + checksum_hash="$(sha1sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)" + fi + cr_data+=("x-amz-checksum-sha1:$checksum_hash") elif [ "$checksum_type" == "crc32" ]; then if [ -z "$checksum_hash" ]; then checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)" fi cr_data+=("x-amz-checksum-crc32:$checksum_hash") elif [ "$checksum_type" == "crc64nvme" ]; then - if [ -z "$checksum_hash" ]; then - if ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then - log_rest 2 "error calculating crc64nvme checksum: $checksum_hash" - exit 1 - fi + if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc64nvme" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then + log_rest 2 "error calculating crc64nvme checksum: $checksum_hash" + exit 1 fi cr_data+=("x-amz-checksum-crc64nvme:$checksum_hash") +elif [ "$checksum_type" == "crc32c" ]; then + if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc32c" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then + log_rest 2 "error calculating crc32c checksum: $checksum_hash" + exit 1 + fi + cr_data+=("x-amz-checksum-crc32c:$checksum_hash") +elif [ "$checksum_type" != "" ]; then + cr_data+=("x-amz-checksum-$checksum_type:$checksum_hash") fi cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time") build_canonical_request "${cr_data[@]}" diff --git a/tests/test_rest.sh b/tests/test_rest.sh index d477703f..8a3906ad 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -375,7 +375,7 @@ test_file="test_file" @test "REST - head object" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1018" + skip "https://github.com/versity/versitygw/issues/1114" fi run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" assert_success @@ -449,9 +449,6 @@ test_file="test_file" } @test "REST - HeadObject does not return 405 with versioning, after file deleted" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1029" - fi if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then skip fi diff --git a/tests/test_rest_checksum.sh b/tests/test_rest_checksum.sh index f5f5c589..185566fe 100755 --- a/tests/test_rest_checksum.sh +++ b/tests/test_rest_checksum.sh @@ -23,34 +23,59 @@ source ./tests/util/util_setup.sh test_file="test_file" -@test "REST - HeadObject returns x-amz-checksum-sha256" { +@test "REST - invalid checksum type" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1104" + fi run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" assert_success - run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256" + run check_invalid_checksum_type "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" assert_success +} - run check_checksum_rest_sha256 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file" +@test "REST - sha256 checksum - invalid" { + run check_checksum_rest_invalid "sha256" assert_success } -@test "REST - PutObject rejects invalid sha256 checksum" { - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" +@test "REST - sha256 checksum - incorrect" { + run check_checksum_rest_incorrect "sha256" assert_success +} - run put_object_rest_sha256_invalid "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" +@test "REST - sha256 checksum - correct" { + run add_correct_checksum "sha256" assert_success } -@test "REST - PutObject rejects incorrect sha256 checksum" { +@test "REST - HeadObject returns x-amz-checksum-sha256" { run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" assert_success - run put_object_rest_sha256_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256" + assert_success + + run check_checksum_rest_sha256 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file" + assert_success +} + +@test "REST - crc32 checksum - invalid" { + run check_checksum_rest_invalid "crc32c" + assert_success +} + +@test "REST - crc32 checksum - incorrect" { + run check_checksum_rest_incorrect "crc32" assert_success } @test "REST - crc32 checksum - correct" { + run add_correct_checksum "crc32" + assert_success +} + +@test "REST - crc32 checksum - HeadObject" { run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" assert_success @@ -61,26 +86,47 @@ test_file="test_file" assert_success } -@test "REST - crc32 checksum - incorrect" { - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" +@test "REST - crc64nvme checksum - invalid" { + run check_checksum_rest_invalid "crc64nvme" assert_success +} - run put_object_rest_crc32_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" +@test "REST - crc64nvme checksum - incorrect" { + run check_checksum_rest_incorrect "crc64nvme" assert_success } @test "REST - crc64nvme checksum - correct" { - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + run add_correct_checksum "sha256" assert_success +} - run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc64nvme" +@test "REST - crc32c checksum - invalid" { + run check_checksum_rest_invalid "crc32c" assert_success } -@test "REST - crc64nvme checksum - incorrect" { - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" +@test "REST - crc32c checksum - incorrect" { + run check_checksum_rest_incorrect "crc32c" assert_success +} + +@test "REST - crc32c checksum - correct" { + run add_correct_checksum "crc32c" + assert_success +} + +@test "REST - sha1 checksum - invalid" { + run check_checksum_rest_invalid "sha1" + assert_success +} + +@test "REST - sha1 checksum - incorrect" { + run check_checksum_rest_incorrect "sha1" + assert_success +} - run put_object_rest_crc64nvme_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" +@test "REST - sha1 checksum - correct" { + run add_correct_checksum "sha1" assert_success } diff --git a/tests/test_s3api_object.sh b/tests/test_s3api_object.sh index bc48d587..a31e253e 100755 --- a/tests/test_s3api_object.sh +++ b/tests/test_s3api_object.sh @@ -72,9 +72,6 @@ export RUN_USERS=true # delete-objects @test "test_delete_objects" { - if [ "$RECREATE_BUCKETS" == "false" ]; then - skip "https://github.com/versity/versitygw/issues/1029" - fi test_delete_objects_s3api_root } @@ -128,16 +125,10 @@ export RUN_USERS=true # test adding and removing an object on versitygw @test "test_put_object_with_data" { - if [ "$RECREATE_BUCKETS" == "false" ]; then - skip "https://github.com/versity/versitygw/issues/1029" - fi test_common_put_object_with_data "s3api" } @test "test_put_object_no_data" { - if [ "$RECREATE_BUCKETS" == "false" ]; then - skip "https://github.com/versity/versitygw/issues/1029" - fi test_common_put_object_no_data "s3api" } diff --git a/tests/util/util_head_object.sh b/tests/util/util_head_object.sh index f7381120..3814f6ac 100644 --- a/tests/util/util_head_object.sh +++ b/tests/util/util_head_object.sh @@ -166,13 +166,3 @@ check_checksum_rest_crc32() { fi return 0 } - -crc64_file() { - source ./env/bin/activate - python3 -m pip install awscrt - if ! checksum=$(python3 -c "import sys;from awscrt import checksums;crc64nvme_value = checksums.crc64nvme(sys.argv[1]);print(crc64nvme_value)" "$1"); then - return 1 - fi - log 5 "checksum: $checksum" - deactivate -} diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh index 0118abd4..645ce9e0 100644 --- a/tests/util/util_multipart.sh +++ b/tests/util/util_multipart.sh @@ -14,6 +14,33 @@ # specific language governing permissions and limitations # under the License. +multipart_upload_s3api_complete_from_bucket() { + if [ $# -ne 3 ]; then + log 2 "'multipart_upload_s3api_complete_from_bucket' requires bucket, copy source, part count" + return 1 + fi + parts="[" + for ((i = 1; i <= $3; i++)); do + # shellcheck disable=SC2154 + if ! upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i"; then + log 2 "error uploading part $i" + return 1 + fi + # shellcheck disable=SC2154 + parts+="{\"ETag\": $etag, \"PartNumber\": $i}" + if [[ $i -ne $3 ]]; then + parts+="," + fi + done + parts+="]" + + if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}' 2>&1); then + log 2 "Error completing upload: $error" + return 1 + fi + return 0 +} + multipart_upload_from_bucket() { if [ $# -ne 4 ]; then log 2 "multipart upload from bucket command missing bucket, copy source, key, and/or part count" @@ -38,23 +65,8 @@ multipart_upload_from_bucket() { return 1 fi - parts="[" - for ((i = 1; i <= $4; i++)); do - # shellcheck disable=SC2154 - if ! upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i"; then - log 2 "error uploading part $i" - return 1 - fi - # shellcheck disable=SC2154 - parts+="{\"ETag\": $etag, \"PartNumber\": $i}" - if [[ $i -ne $4 ]]; then - parts+="," - fi - done - parts+="]" - - if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}' 2>&1); then - log 2 "Error completing upload: $error" + if ! multipart_upload_s3api_complete_from_bucket "$1" "$2" "$4"; then + log 2 " error completing multipart upload from bucket" return 1 fi return 0 @@ -181,8 +193,6 @@ multipart_upload_with_params() { return 0 } - - run_and_verify_multipart_upload_with_valid_range() { if [ $# -ne 3 ]; then log 2 "'run_and_verify_multipart_upload_with_valid_range' requires bucket, key, 5MB file" diff --git a/tests/util/util_object.sh b/tests/util/util_object.sh index 9e899537..62e6e062 100644 --- a/tests/util/util_object.sh +++ b/tests/util/util_object.sh @@ -307,7 +307,7 @@ check_checksum_invalid_or_incorrect() { return 1 fi if [ "$result" != "400" ]; then - log 2 "expected response code of '400', was '$result' (response: $(cat "$TEST_FILE_FOLDER/result.txt")" + log 2 "expected response code of '400', was '$result' (response: $(cat "$TEST_FILE_FOLDER/result.txt"))" return 1 fi if ! check_xml_element "$TEST_FILE_FOLDER/result.txt" "$6" "Error" "Message"; then @@ -331,40 +331,87 @@ put_object_rest_checksum() { log 2 "expected response code of '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" return 1 fi - log 5 "result: $(cat "$TEST_FILE_FOLDER/result.txt")" return 0 } -put_object_rest_sha256_invalid() { - if [ $# -ne 3 ]; then - log 2 "'put_object_rest_sha256_invalid' requires data file, bucket name, key" +check_checksum_rest_invalid() { + if [ $# -ne 1 ]; then + log 2 "'put_object_rest_sha256_invalid' requires checksum type" + return 1 + fi + test_file="test_file" + if ! setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"; then + log 2 "error setting up bucket and file" return 1 fi - if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256" "dummy" "Value for x-amz-checksum-sha256 header is invalid."; then + if ! check_checksum_invalid_or_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$1" "dummy" "Value for x-amz-checksum-$1 header is invalid."; then log 2 "error checking checksum" return 1 fi return 0 } -put_object_rest_sha256_incorrect() { - if [ $# -ne 3 ]; then - log 2 "'put_object_rest_sha256_invalid' requires data file, bucket name, key" +check_checksum_rest_incorrect() { + if [ $# -ne 1 ]; then + log 2 "'check_checksum_rest_incorrect' requires checksum type" + return 1 + fi + test_file="test_file" + if ! setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"; then + log 2 "error setting up bucket and file" return 1 fi if [ "$DIRECT" == "true" ]; then - error_message="The SHA256 you specified did not match the calculated checksum." + error_cs_str="$(echo "$1" | tr '[:lower:]' '[:upper:]')" else - error_message="The sha256 you specified did not match the calculated checksum." + error_cs_str="$1" fi - incorrect_checksum="$(echo -n "dummy" | sha256sum | awk '{print $1}' | xxd -r -p | base64)" - if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256" "$incorrect_checksum" "$error_message"; then + error_message="The $error_cs_str you specified did not match the calculated checksum." + if ! calculate_incorrect_checksum "$1" "$(cat "$TEST_FILE_FOLDER/$test_file")"; then + log 2 "error calculating incorrect checksum" + return 1 + fi + if ! check_checksum_invalid_or_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$1" "$incorrect_checksum" "$error_message"; then log 2 "error checking checksum" return 1 fi return 0 } +calculate_incorrect_checksum() { + if [ $# -ne 2 ]; then + log 2 "'calculate_incorrect_checksum' requires checksum type, data" + return 1 + fi + case "$1" in + "sha1") + incorrect_checksum="$(echo -n "$2"a | sha1sum | awk '{print $1}' | xxd -r -p | base64)" + ;; + "sha256") + incorrect_checksum="$(echo -n "$2"a | sha256sum | awk '{print $1}' | xxd -r -p | base64)" + ;; + "crc32") + incorrect_checksum="$(echo -n "$2"a | gzip -c -1 | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)" + ;; + "crc32c") + if ! incorrect_checksum=$(DATA_FILE=<(echo -n "$2"a) TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc32c" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then + log 2 "error calculating checksum: $incorrect_checksum" + return 1 + fi + ;; + "crc64nvme") + if ! incorrect_checksum=$(DATA_FILE=<(echo -n "$2"a) TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc64nvme" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then + log 2 "error calculating checksum: $incorrect_checksum" + return 1 + fi + ;; + *) + log 2 "invalid checksum type: $1" + return 1 + esac + return 0 +} + put_object_rest_chunked_payload_type_without_content_length() { if [ $# -ne 3 ]; then log 2 "'put_object_rest_diff_payload_type' requires data file, bucket name, key" @@ -381,41 +428,32 @@ put_object_rest_chunked_payload_type_without_content_length() { return 0 } -put_object_rest_crc32_incorrect() { - if [ $# -ne 3 ]; then - log 2 "'put_object_rest_crc32_incorrect' requires data file, bucket name, key" +add_correct_checksum() { + if [ $# -ne 1 ]; then + log 2 "'add_correct_checksum' requires checksum type" return 1 fi - if [ "$DIRECT" == "true" ]; then - error_message="The CRC32 you specified did not match the calculated checksum." - else - error_message="The crc32 you specified did not match the calculated checksum." + test_file="test_file" + if ! setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"; then + log 2 "error setting up bucket and file" + return 1 fi - incorrect_checksum="$(echo -n "dummy" | gzip -c -1 | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)" - if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "crc32" "$incorrect_checksum" "$error_message"; then - log 2 "error checking checksum" + + if ! put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$1"; then + log 2 "error adding file with checksum to s3" return 1 fi return 0 } -put_object_rest_crc64nvme_incorrect() { +check_invalid_checksum_type() { if [ $# -ne 3 ]; then - log 2 "'put_object_rest_crc64nvme_incorrect' requires data file, bucket name, key" - return 1 - fi - if [ "$DIRECT" == "true" ]; then - error_message="The CRC64NVME you specified did not match the calculated checksum." - else - error_message="The crc64nvme you specified did not match the calculated checksum." - fi - if ! incorrect_checksum=$(DATA_FILE=<(echo -n "dummy") TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then - log 2 "error calculating checksum: $incorrect_checksum" + log 2 "'check_invalid_checksum_type' requires data file, bucket name, file" return 1 fi - if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "crc64nvme" "$incorrect_checksum" "$error_message"; then + error_message='The algorithm type you specified in x-amz-checksum- header is invalid.' + if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256a" "dummy" "$error_message"; then log 2 "error checking checksum" return 1 fi - return 0 }