Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test/rest checksum crc32c #1115

Merged
merged 2 commits into from
Mar 10, 2025
+231 −124
Merged

Test/rest checksum crc32c #1115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4c5cd91
test: crc64nvme checksum
lrm25 Feb 27, 2025
485f2ab
test: crc32c checksum
lrm25 Mar 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 41 additions & 33 deletions tests/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,29 +58,7 @@ source_config_file() {
fi
}

check_universal_vars() {
if [[ $BYPASS_ENV_FILE != "true" ]]; then
source_config_file
fi
if [ -n "$COMMAND_LOG" ]; then
init_command_log
fi

if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
# shellcheck source=./tests/.secrets
source "$SECRETS_FILE"
else
log 3 "Warning: no secrets file found"
fi

if [[ -n "$LOG_LEVEL" ]]; then
if [[ $LOG_LEVEL -lt 2 ]]; then
log 1 "log level must be 2 or greater"
exit 1
fi
export LOG_LEVEL_INT=$LOG_LEVEL
fi

check_aws_vars() {
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
log 1 "AWS_ACCESS_KEY_ID missing"
exit 1
Expand All @@ -103,10 +81,14 @@ check_universal_vars() {
exit 1
fi
fi
if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
fail "RUN_VERSITYGW must be 'true' or 'false'"
# exporting these since they're needed for subshells
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
if [ -n "$AWS_CANONICAL_ID" ]; then
export AWS_CANONICAL_ID
fi
}

check_bucket_vars() {
if [ -z "$BUCKET_ONE_NAME" ]; then
log 1 "BUCKET_ONE_NAME missing"
exit 1
Expand All @@ -129,24 +111,50 @@ check_universal_vars() {
fi
if [ "$RECREATE_BUCKETS" == "false" ] && [ "$DELETE_BUCKETS_AFTER_TEST" == "true" ]; then
log 1 "cannot set DELETE_BUCKETS_AFTER_TEST to 'true' if RECREATE_BUCKETS is 'false'"
return 1
exit 1
fi
}

check_universal_vars() {
if [[ $BYPASS_ENV_FILE != "true" ]]; then
source_config_file
fi
if [ -n "$COMMAND_LOG" ]; then
init_command_log
fi
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
# shellcheck source=./tests/.secrets
source "$SECRETS_FILE"
else
log 3 "Warning: no secrets file found"
fi
if [[ -n "$LOG_LEVEL" ]]; then
if [[ $LOG_LEVEL -lt 2 ]]; then
log 1 "log level must be 2 or greater"
exit 1
fi
export LOG_LEVEL_INT=$LOG_LEVEL
fi

check_aws_vars

if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
log 1 "RUN_VERSITYGW must be 'true' or 'false'"
exit 1
fi

check_bucket_vars

if [ -z "$TEST_FILE_FOLDER" ]; then
log 1 "TEST_FILE_FOLDER missing"
exit 1
fi
if [ ! -d "$TEST_FILE_FOLDER" ]; then
if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then
log 2 "error creating test folder: $error"
log 1 "error creating test folder: $error"
exit 1
fi
fi
# exporting these since they're needed for subshells
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
if [ -n "$AWS_CANONICAL_ID" ]; then
log 5 "canonical ID: $AWS_CANONICAL_ID"
export AWS_CANONICAL_ID
fi
}

delete_command_log() {
Expand Down
10 changes: 8 additions & 2 deletions tests/rest_scripts/calculate_crc64nvme.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import sys
from awscrt import checksums
with open(sys.argv[1], 'rb') as f:
print(checksums.crc64nvme(f.read()))" "$DATA_FILE" 2>&1); then
print(checksums.${CHECKSUM_TYPE}(f.read()))" "$DATA_FILE" 2>&1); then
log_rest 2 "error calculating checksum: $checksum_decimal"
exit 1
fi
Expand All @@ -34,5 +34,11 @@ if ! deactivate 1>/dev/null; then
log_rest 2 "error deactivating virtual environment"
exit 1
fi
checksum_hash=$(printf "%016x" "$checksum_decimal" | xxd -r -p | base64)
if [ "$CHECKSUM_TYPE" == "crc64nvme" ]; then
hex_format="%016x"
else
hex_format="%08x"
fi
# shellcheck disable=SC2059
checksum_hash=$(printf "$hex_format" "$checksum_decimal" | xxd -r -p | base64)
echo "$checksum_hash"
21 changes: 16 additions & 5 deletions tests/rest_scripts/put_object.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,19 +46,30 @@ if [ "$checksum_type" == "sha256" ]; then
checksum_hash="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
fi
cr_data+=("x-amz-checksum-sha256:$checksum_hash")
elif [ "$checksum_type" == "sha1" ]; then
if [ -z "$checksum_hash" ]; then
checksum_hash="$(sha1sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
fi
cr_data+=("x-amz-checksum-sha1:$checksum_hash")
elif [ "$checksum_type" == "crc32" ]; then
if [ -z "$checksum_hash" ]; then
checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
fi
cr_data+=("x-amz-checksum-crc32:$checksum_hash")
elif [ "$checksum_type" == "crc64nvme" ]; then
if [ -z "$checksum_hash" ]; then
if ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
exit 1
fi
if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc64nvme" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
exit 1
fi
cr_data+=("x-amz-checksum-crc64nvme:$checksum_hash")
elif [ "$checksum_type" == "crc32c" ]; then
if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc32c" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
log_rest 2 "error calculating crc32c checksum: $checksum_hash"
exit 1
fi
cr_data+=("x-amz-checksum-crc32c:$checksum_hash")
elif [ "$checksum_type" != "" ]; then
cr_data+=("x-amz-checksum-$checksum_type:$checksum_hash")
fi
cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
build_canonical_request "${cr_data[@]}"
Expand Down
5 changes: 1 addition & 4 deletions tests/test_rest.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ test_file="test_file"

@test "REST - head object" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1018"
skip "https://github.com/versity/versitygw/issues/1114"
fi
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
Expand Down Expand Up @@ -449,9 +449,6 @@ test_file="test_file"
}

@test "REST - HeadObject does not return 405 with versioning, after file deleted" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1029"
fi
if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
skip
fi
Expand Down
78 changes: 62 additions & 16 deletions tests/test_rest_checksum.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,34 +23,59 @@ source ./tests/util/util_setup.sh

test_file="test_file"

@test "REST - HeadObject returns x-amz-checksum-sha256" {
@test "REST - invalid checksum type" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1104"
fi
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success

run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256"
run check_invalid_checksum_type "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success
}

run check_checksum_rest_sha256 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
@test "REST - sha256 checksum - invalid" {
run check_checksum_rest_invalid "sha256"
assert_success
}

@test "REST - PutObject rejects invalid sha256 checksum" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
@test "REST - sha256 checksum - incorrect" {
run check_checksum_rest_incorrect "sha256"
assert_success
}

run put_object_rest_sha256_invalid "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@test "REST - sha256 checksum - correct" {
run add_correct_checksum "sha256"
assert_success
}

@test "REST - PutObject rejects incorrect sha256 checksum" {
@test "REST - HeadObject returns x-amz-checksum-sha256" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success

run put_object_rest_sha256_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256"
assert_success

run check_checksum_rest_sha256 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
assert_success
}

@test "REST - crc32 checksum - invalid" {
run check_checksum_rest_invalid "crc32c"
assert_success
}

@test "REST - crc32 checksum - incorrect" {
run check_checksum_rest_incorrect "crc32"
assert_success
}

@test "REST - crc32 checksum - correct" {
run add_correct_checksum "crc32"
assert_success
}

@test "REST - crc32 checksum - HeadObject" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success

Expand All @@ -61,26 +86,47 @@ test_file="test_file"
assert_success
}

@test "REST - crc32 checksum - incorrect" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
@test "REST - crc64nvme checksum - invalid" {
run check_checksum_rest_invalid "crc64nvme"
assert_success
}

run put_object_rest_crc32_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@test "REST - crc64nvme checksum - incorrect" {
run check_checksum_rest_incorrect "crc64nvme"
assert_success
}

@test "REST - crc64nvme checksum - correct" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
run add_correct_checksum "sha256"
assert_success
}

run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc64nvme"
@test "REST - crc32c checksum - invalid" {
run check_checksum_rest_invalid "crc32c"
assert_success
}

@test "REST - crc64nvme checksum - incorrect" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
@test "REST - crc32c checksum - incorrect" {
run check_checksum_rest_incorrect "crc32c"
assert_success
}

@test "REST - crc32c checksum - correct" {
run add_correct_checksum "crc32c"
assert_success
}

@test "REST - sha1 checksum - invalid" {
run check_checksum_rest_invalid "sha1"
assert_success
}

@test "REST - sha1 checksum - incorrect" {
run check_checksum_rest_incorrect "sha1"
assert_success
}

run put_object_rest_crc64nvme_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@test "REST - sha1 checksum - correct" {
run add_correct_checksum "sha1"
assert_success
}
9 changes: 0 additions & 9 deletions tests/test_s3api_object.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,6 @@ export RUN_USERS=true

# delete-objects
@test "test_delete_objects" {
if [ "$RECREATE_BUCKETS" == "false" ]; then
skip "https://github.com/versity/versitygw/issues/1029"
fi
test_delete_objects_s3api_root
}

Expand Down Expand Up @@ -128,16 +125,10 @@ export RUN_USERS=true

# test adding and removing an object on versitygw
@test "test_put_object_with_data" {
if [ "$RECREATE_BUCKETS" == "false" ]; then
skip "https://github.com/versity/versitygw/issues/1029"
fi
test_common_put_object_with_data "s3api"
}

@test "test_put_object_no_data" {
if [ "$RECREATE_BUCKETS" == "false" ]; then
skip "https://github.com/versity/versitygw/issues/1029"
fi
test_common_put_object_no_data "s3api"
}

Expand Down
Loading