Skip to content

Commit 485f2ab

Browse files
lrm25lrm25
committed
test: crc32c checksum
1 parent 4c5cd91 commit 485f2ab

9 files changed

+231
-134
lines changed

‎tests/env.sh

+41-33
Original file line numberDiff line numberDiff line change
@@ -58,29 +58,7 @@ source_config_file() {
5858
fi
5959
}
6060

61-
check_universal_vars() {
62-
if [[ $BYPASS_ENV_FILE != "true" ]]; then
63-
source_config_file
64-
fi
65-
if [ -n "$COMMAND_LOG" ]; then
66-
init_command_log
67-
fi
68-
69-
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
70-
# shellcheck source=./tests/.secrets
71-
source "$SECRETS_FILE"
72-
else
73-
log 3 "Warning: no secrets file found"
74-
fi
75-
76-
if [[ -n "$LOG_LEVEL" ]]; then
77-
if [[ $LOG_LEVEL -lt 2 ]]; then
78-
log 1 "log level must be 2 or greater"
79-
exit 1
80-
fi
81-
export LOG_LEVEL_INT=$LOG_LEVEL
82-
fi
83-
61+
check_aws_vars() {
8462
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
8563
log 1 "AWS_ACCESS_KEY_ID missing"
8664
exit 1
@@ -103,10 +81,14 @@ check_universal_vars() {
10381
exit 1
10482
fi
10583
fi
106-
if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
107-
fail "RUN_VERSITYGW must be 'true' or 'false'"
84+
# exporting these since they're needed for subshells
85+
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
86+
if [ -n "$AWS_CANONICAL_ID" ]; then
87+
export AWS_CANONICAL_ID
10888
fi
89+
}
10990

91+
check_bucket_vars() {
11092
if [ -z "$BUCKET_ONE_NAME" ]; then
11193
log 1 "BUCKET_ONE_NAME missing"
11294
exit 1
@@ -129,24 +111,50 @@ check_universal_vars() {
129111
fi
130112
if [ "$RECREATE_BUCKETS" == "false" ] && [ "$DELETE_BUCKETS_AFTER_TEST" == "true" ]; then
131113
log 1 "cannot set DELETE_BUCKETS_AFTER_TEST to 'true' if RECREATE_BUCKETS is 'false'"
132-
return 1
114+
exit 1
115+
fi
116+
}
117+
118+
check_universal_vars() {
119+
if [[ $BYPASS_ENV_FILE != "true" ]]; then
120+
source_config_file
121+
fi
122+
if [ -n "$COMMAND_LOG" ]; then
123+
init_command_log
124+
fi
125+
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
126+
# shellcheck source=./tests/.secrets
127+
source "$SECRETS_FILE"
128+
else
129+
log 3 "Warning: no secrets file found"
130+
fi
131+
if [[ -n "$LOG_LEVEL" ]]; then
132+
if [[ $LOG_LEVEL -lt 2 ]]; then
133+
log 1 "log level must be 2 or greater"
134+
exit 1
135+
fi
136+
export LOG_LEVEL_INT=$LOG_LEVEL
137+
fi
138+
139+
check_aws_vars
140+
141+
if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
142+
log 1 "RUN_VERSITYGW must be 'true' or 'false'"
143+
exit 1
133144
fi
145+
146+
check_bucket_vars
147+
134148
if [ -z "$TEST_FILE_FOLDER" ]; then
135149
log 1 "TEST_FILE_FOLDER missing"
136150
exit 1
137151
fi
138152
if [ ! -d "$TEST_FILE_FOLDER" ]; then
139153
if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then
140-
log 2 "error creating test folder: $error"
154+
log 1 "error creating test folder: $error"
141155
exit 1
142156
fi
143157
fi
144-
# exporting these since they're needed for subshells
145-
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
146-
if [ -n "$AWS_CANONICAL_ID" ]; then
147-
log 5 "canonical ID: $AWS_CANONICAL_ID"
148-
export AWS_CANONICAL_ID
149-
fi
150158
}
151159

152160
delete_command_log() {

‎tests/rest_scripts/calculate_crc64nvme.sh

+8-2
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import sys
2525
from awscrt import checksums
2626
2727
with open(sys.argv[1], 'rb') as f:
28-
print(checksums.crc64nvme(f.read()))" "$DATA_FILE" 2>&1); then
28+
print(checksums.${CHECKSUM_TYPE}(f.read()))" "$DATA_FILE" 2>&1); then
2929
log_rest 2 "error calculating checksum: $checksum_decimal"
3030
exit 1
3131
fi
@@ -34,5 +34,11 @@ if ! deactivate 1>/dev/null; then
3434
log_rest 2 "error deactivating virtual environment"
3535
exit 1
3636
fi
37-
checksum_hash=$(printf "%016x" "$checksum_decimal" | xxd -r -p | base64)
37+
if [ "$CHECKSUM_TYPE" == "crc64nvme" ]; then
38+
hex_format="%016x"
39+
else
40+
hex_format="%08x"
41+
fi
42+
# shellcheck disable=SC2059
43+
checksum_hash=$(printf "$hex_format" "$checksum_decimal" | xxd -r -p | base64)
3844
echo "$checksum_hash"

‎tests/rest_scripts/put_object.sh

+16-5
Original file line numberDiff line numberDiff line change
@@ -46,19 +46,30 @@ if [ "$checksum_type" == "sha256" ]; then
4646
checksum_hash="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
4747
fi
4848
cr_data+=("x-amz-checksum-sha256:$checksum_hash")
49+
elif [ "$checksum_type" == "sha1" ]; then
50+
if [ -z "$checksum_hash" ]; then
51+
checksum_hash="$(sha1sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
52+
fi
53+
cr_data+=("x-amz-checksum-sha1:$checksum_hash")
4954
elif [ "$checksum_type" == "crc32" ]; then
5055
if [ -z "$checksum_hash" ]; then
5156
checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
5257
fi
5358
cr_data+=("x-amz-checksum-crc32:$checksum_hash")
5459
elif [ "$checksum_type" == "crc64nvme" ]; then
55-
if [ -z "$checksum_hash" ]; then
56-
if ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
57-
log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
58-
exit 1
59-
fi
60+
if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc64nvme" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
61+
log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
62+
exit 1
6063
fi
6164
cr_data+=("x-amz-checksum-crc64nvme:$checksum_hash")
65+
elif [ "$checksum_type" == "crc32c" ]; then
66+
if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc32c" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
67+
log_rest 2 "error calculating crc32c checksum: $checksum_hash"
68+
exit 1
69+
fi
70+
cr_data+=("x-amz-checksum-crc32c:$checksum_hash")
71+
elif [ "$checksum_type" != "" ]; then
72+
cr_data+=("x-amz-checksum-$checksum_type:$checksum_hash")
6273
fi
6374
cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
6475
build_canonical_request "${cr_data[@]}"

‎tests/test_rest.sh

+1-4
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,7 @@ test_file="test_file"
375375

376376
@test "REST - head object" {
377377
if [ "$DIRECT" != "true" ]; then
378-
skip "https://github.com/versity/versitygw/issues/1018"
378+
skip "https://github.com/versity/versitygw/issues/1114"
379379
fi
380380
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
381381
assert_success
@@ -449,9 +449,6 @@ test_file="test_file"
449449
}
450450

451451
@test "REST - HeadObject does not return 405 with versioning, after file deleted" {
452-
if [ "$DIRECT" != "true" ]; then
453-
skip "https://github.com/versity/versitygw/issues/1029"
454-
fi
455452
if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
456453
skip
457454
fi

‎tests/test_rest_checksum.sh

+62-16
Original file line numberDiff line numberDiff line change
@@ -23,34 +23,59 @@ source ./tests/util/util_setup.sh
2323

2424
test_file="test_file"
2525

26-
@test "REST - HeadObject returns x-amz-checksum-sha256" {
26+
@test "REST - invalid checksum type" {
27+
if [ "$DIRECT" != "true" ]; then
28+
skip "https://github.com/versity/versitygw/issues/1104"
29+
fi
2730
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
2831
assert_success
2932

30-
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256"
33+
run check_invalid_checksum_type "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
3134
assert_success
35+
}
3236

33-
run check_checksum_rest_sha256 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
37+
@test "REST - sha256 checksum - invalid" {
38+
run check_checksum_rest_invalid "sha256"
3439
assert_success
3540
}
3641

37-
@test "REST - PutObject rejects invalid sha256 checksum" {
38-
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
42+
@test "REST - sha256 checksum - incorrect" {
43+
run check_checksum_rest_incorrect "sha256"
3944
assert_success
45+
}
4046

41-
run put_object_rest_sha256_invalid "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
47+
@test "REST - sha256 checksum - correct" {
48+
run add_correct_checksum "sha256"
4249
assert_success
4350
}
4451

45-
@test "REST - PutObject rejects incorrect sha256 checksum" {
52+
@test "REST - HeadObject returns x-amz-checksum-sha256" {
4653
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
4754
assert_success
4855

49-
run put_object_rest_sha256_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
56+
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256"
57+
assert_success
58+
59+
run check_checksum_rest_sha256 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
60+
assert_success
61+
}
62+
63+
@test "REST - crc32 checksum - invalid" {
64+
run check_checksum_rest_invalid "crc32c"
65+
assert_success
66+
}
67+
68+
@test "REST - crc32 checksum - incorrect" {
69+
run check_checksum_rest_incorrect "crc32"
5070
assert_success
5171
}
5272

5373
@test "REST - crc32 checksum - correct" {
74+
run add_correct_checksum "crc32"
75+
assert_success
76+
}
77+
78+
@test "REST - crc32 checksum - HeadObject" {
5479
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
5580
assert_success
5681

@@ -61,26 +86,47 @@ test_file="test_file"
6186
assert_success
6287
}
6388

64-
@test "REST - crc32 checksum - incorrect" {
65-
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
89+
@test "REST - crc64nvme checksum - invalid" {
90+
run check_checksum_rest_invalid "crc64nvme"
6691
assert_success
92+
}
6793

68-
run put_object_rest_crc32_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
94+
@test "REST - crc64nvme checksum - incorrect" {
95+
run check_checksum_rest_incorrect "crc64nvme"
6996
assert_success
7097
}
7198

7299
@test "REST - crc64nvme checksum - correct" {
73-
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
100+
run add_correct_checksum "sha256"
74101
assert_success
102+
}
75103

76-
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc64nvme"
104+
@test "REST - crc32c checksum - invalid" {
105+
run check_checksum_rest_invalid "crc32c"
77106
assert_success
78107
}
79108

80-
@test "REST - crc64nvme checksum - incorrect" {
81-
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
109+
@test "REST - crc32c checksum - incorrect" {
110+
run check_checksum_rest_incorrect "crc32c"
82111
assert_success
112+
}
113+
114+
@test "REST - crc32c checksum - correct" {
115+
run add_correct_checksum "crc32c"
116+
assert_success
117+
}
118+
119+
@test "REST - sha1 checksum - invalid" {
120+
run check_checksum_rest_invalid "sha1"
121+
assert_success
122+
}
123+
124+
@test "REST - sha1 checksum - incorrect" {
125+
run check_checksum_rest_incorrect "sha1"
126+
assert_success
127+
}
83128

84-
run put_object_rest_crc64nvme_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
129+
@test "REST - sha1 checksum - correct" {
130+
run add_correct_checksum "sha1"
85131
assert_success
86132
}

‎tests/test_s3api_object.sh

-9
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,6 @@ export RUN_USERS=true
7272

7373
# delete-objects
7474
@test "test_delete_objects" {
75-
if [ "$RECREATE_BUCKETS" == "false" ]; then
76-
skip "https://github.com/versity/versitygw/issues/1029"
77-
fi
7875
test_delete_objects_s3api_root
7976
}
8077

@@ -128,16 +125,10 @@ export RUN_USERS=true
128125

129126
# test adding and removing an object on versitygw
130127
@test "test_put_object_with_data" {
131-
if [ "$RECREATE_BUCKETS" == "false" ]; then
132-
skip "https://github.com/versity/versitygw/issues/1029"
133-
fi
134128
test_common_put_object_with_data "s3api"
135129
}
136130

137131
@test "test_put_object_no_data" {
138-
if [ "$RECREATE_BUCKETS" == "false" ]; then
139-
skip "https://github.com/versity/versitygw/issues/1029"
140-
fi
141132
test_common_put_object_no_data "s3api"
142133
}
143134

‎tests/util/util_head_object.sh

-10
Original file line numberDiff line numberDiff line change
@@ -166,13 +166,3 @@ check_checksum_rest_crc32() {
166166
fi
167167
return 0
168168
}
169-
170-
crc64_file() {
171-
source ./env/bin/activate
172-
python3 -m pip install awscrt
173-
if ! checksum=$(python3 -c "import sys;from awscrt import checksums;crc64nvme_value = checksums.crc64nvme(sys.argv[1]);print(crc64nvme_value)" "$1"); then
174-
return 1
175-
fi
176-
log 5 "checksum: $checksum"
177-
deactivate
178-
}

0 commit comments

Comments
 (0)