Skip to content

Commit 6db2aeb

Browse files
lrm25lrm25
committed
test: checksum mode test
1 parent 1d75da5 commit 6db2aeb

21 files changed

+366
-203
lines changed

‎tests/commands/head_bucket.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,13 @@ head_bucket() {
3939
log 2 "invalid command type $1"
4040
fi
4141
if [ $exit_code -ne 0 ]; then
42-
log 2 "error getting bucket info: $bucket_info"
4342
if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then
4443
return 1
4544
fi
45+
log 2 "error getting bucket info: $bucket_info"
4646
return 2
4747
fi
48+
bucket_info="$(echo -n "$bucket_info" | grep -v "InsecureRequestWarning")"
4849
echo "$bucket_info"
4950
return 0
5051
}

‎tests/commands/put_bucket_acl.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ put_bucket_acl_s3api() {
2424
log 2 "put bucket acl command requires bucket name, acl file"
2525
return 1
2626
fi
27-
log 5 "bucket name: $1, acls: $2"
27+
log 5 "bucket name: $1, acls: $(cat "$2")"
2828
if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then
2929
log 2 "error putting bucket acl: $error"
3030
return 1

‎tests/env.sh

+39-32
Original file line numberDiff line numberDiff line change
@@ -58,29 +58,7 @@ source_config_file() {
5858
fi
5959
}
6060

61-
check_universal_vars() {
62-
if [[ $BYPASS_ENV_FILE != "true" ]]; then
63-
source_config_file
64-
fi
65-
if [ -n "$COMMAND_LOG" ]; then
66-
init_command_log
67-
fi
68-
69-
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
70-
# shellcheck source=./tests/.secrets
71-
source "$SECRETS_FILE"
72-
else
73-
log 3 "Warning: no secrets file found"
74-
fi
75-
76-
if [[ -n "$LOG_LEVEL" ]]; then
77-
if [[ $LOG_LEVEL -lt 2 ]]; then
78-
log 1 "log level must be 2 or greater"
79-
exit 1
80-
fi
81-
export LOG_LEVEL_INT=$LOG_LEVEL
82-
fi
83-
61+
check_aws_vars() {
8462
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
8563
log 1 "AWS_ACCESS_KEY_ID missing"
8664
exit 1
@@ -103,10 +81,14 @@ check_universal_vars() {
10381
exit 1
10482
fi
10583
fi
106-
if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
107-
fail "RUN_VERSITYGW must be 'true' or 'false'"
84+
# exporting these since they're needed for subshells
85+
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
86+
if [ -n "$AWS_CANONICAL_ID" ]; then
87+
export AWS_CANONICAL_ID
10888
fi
89+
}
10990

91+
check_bucket_vars() {
11092
if [ -z "$BUCKET_ONE_NAME" ]; then
11193
log 1 "BUCKET_ONE_NAME missing"
11294
exit 1
@@ -129,8 +111,39 @@ check_universal_vars() {
129111
fi
130112
if [ "$RECREATE_BUCKETS" == "false" ] && [ "$DELETE_BUCKETS_AFTER_TEST" == "true" ]; then
131113
log 1 "cannot set DELETE_BUCKETS_AFTER_TEST to 'true' if RECREATE_BUCKETS is 'false'"
132-
return 1
114+
exit 1
133115
fi
116+
}
117+
118+
check_universal_vars() {
119+
if [[ $BYPASS_ENV_FILE != "true" ]]; then
120+
source_config_file
121+
fi
122+
if [ -n "$COMMAND_LOG" ]; then
123+
init_command_log
124+
fi
125+
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
126+
# shellcheck source=./tests/.secrets
127+
source "$SECRETS_FILE"
128+
else
129+
log 3 "Warning: no secrets file found"
130+
fi
131+
if [[ -n "$LOG_LEVEL" ]]; then
132+
if [[ $LOG_LEVEL -lt 2 ]]; then
133+
log 1 "log level must be 2 or greater"
134+
exit 1
135+
fi
136+
export LOG_LEVEL_INT=$LOG_LEVEL
137+
fi
138+
139+
check_aws_vars
140+
141+
if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
142+
fail "RUN_VERSITYGW must be 'true' or 'false'"
143+
fi
144+
145+
check_bucket_vars
146+
134147
if [ -z "$TEST_FILE_FOLDER" ]; then
135148
log 1 "TEST_FILE_FOLDER missing"
136149
exit 1
@@ -141,12 +154,6 @@ check_universal_vars() {
141154
exit 1
142155
fi
143156
fi
144-
# exporting these since they're needed for subshells
145-
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
146-
if [ -n "$AWS_CANONICAL_ID" ]; then
147-
log 5 "canonical ID: $AWS_CANONICAL_ID"
148-
export AWS_CANONICAL_ID
149-
fi
150157
}
151158

152159
delete_command_log() {

‎tests/rest_scripts/calculate_crc64nvme.sh

+8-2
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import sys
2525
from awscrt import checksums
2626
2727
with open(sys.argv[1], 'rb') as f:
28-
print(checksums.crc64nvme(f.read()))" "$DATA_FILE" 2>&1); then
28+
print(checksums.${CHECKSUM_TYPE}(f.read()))" "$DATA_FILE" 2>&1); then
2929
log_rest 2 "error calculating checksum: $checksum_decimal"
3030
exit 1
3131
fi
@@ -34,5 +34,11 @@ if ! deactivate 1>/dev/null; then
3434
log_rest 2 "error deactivating virtual environment"
3535
exit 1
3636
fi
37-
checksum_hash=$(printf "%016x" "$checksum_decimal" | xxd -r -p | base64)
37+
if [ "$CHECKSUM_TYPE" == "crc64nvme" ]; then
38+
hex_format="%016x"
39+
else
40+
hex_format="%08x"
41+
fi
42+
# shellcheck disable=SC2059
43+
checksum_hash=$(printf "$hex_format" "$checksum_decimal" | xxd -r -p | base64)
3844
echo "$checksum_hash"

‎tests/rest_scripts/put_object.sh

+16-5
Original file line numberDiff line numberDiff line change
@@ -46,19 +46,30 @@ if [ "$checksum_type" == "sha256" ]; then
4646
checksum_hash="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
4747
fi
4848
cr_data+=("x-amz-checksum-sha256:$checksum_hash")
49+
elif [ "$checksum_type" == "sha1" ]; then
50+
if [ -z "$checksum_hash" ]; then
51+
checksum_hash="$(sha1sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
52+
fi
53+
cr_data+=("x-amz-checksum-sha1:$checksum_hash")
4954
elif [ "$checksum_type" == "crc32" ]; then
5055
if [ -z "$checksum_hash" ]; then
5156
checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
5257
fi
5358
cr_data+=("x-amz-checksum-crc32:$checksum_hash")
5459
elif [ "$checksum_type" == "crc64nvme" ]; then
55-
if [ -z "$checksum_hash" ]; then
56-
if ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
57-
log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
58-
exit 1
59-
fi
60+
if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc64nvme" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
61+
log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
62+
exit 1
6063
fi
6164
cr_data+=("x-amz-checksum-crc64nvme:$checksum_hash")
65+
elif [ "$checksum_type" == "crc32c" ]; then
66+
if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc32c" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
67+
log_rest 2 "error calculating crc32c checksum: $checksum_hash"
68+
exit 1
69+
fi
70+
cr_data+=("x-amz-checksum-crc32c:$checksum_hash")
71+
elif [ "$checksum_type" != "" ]; then
72+
cr_data+=("x-amz-checksum-$checksum_type:$checksum_hash")
6273
fi
6374
cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
6475
build_canonical_request "${cr_data[@]}"

‎tests/rest_scripts/put_object_openssl_chunked_example.sh

+9
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,15 @@ fi
322322
command+="$chunks"
323323
command="${command//$'\n'/$'\r\n'}"
324324
echo -n "$command" > "$COMMAND_FILE"
325+
if [ -n "$COMMAND_LOG" ]; then
326+
while IFS= read -r line; do
327+
if ! mask_arg_array "$line"; then
328+
return 1
329+
fi
330+
# shellcheck disable=SC2154
331+
echo "${masked_args[*]}" >> "$COMMAND_LOG"
332+
done <<< "$command"
333+
fi
325334
}
326335

327336
load_parameters

‎tests/test_common_acl.sh

+5-6
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,14 @@ test_put_bucket_acl_s3cmd() {
4646
test_common_put_bucket_acl() {
4747
assert [ $# -eq 1 ]
4848

49-
run setup_bucket "$1" "$BUCKET_ONE_NAME"
49+
run setup_bucket_and_user "$BUCKET_ONE_NAME" "$USERNAME_ONE" "$PASSWORD_ONE" "user"
5050
assert_success
51+
# shellcheck disable=SC2154
52+
username="${lines[${#lines[@]}-2]}"
5153

5254
run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred"
5355
assert_success
5456

55-
run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user"
56-
assert_success
57-
5857
run get_check_acl_id "$1" "$BUCKET_ONE_NAME"
5958
assert_success
6059

@@ -67,7 +66,7 @@ test_common_put_bucket_acl() {
6766
grantee_id="http://acs.amazonaws.com/groups/global/AllUsers"
6867
else
6968
grantee_type="CanonicalUser"
70-
grantee_id="$USERNAME_ONE"
69+
grantee_id="$username"
7170
fi
7271
run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "$grantee_type" "$grantee_id" "READ" "$AWS_ACCESS_KEY_ID"
7372
assert_success
@@ -79,7 +78,7 @@ test_common_put_bucket_acl() {
7978
run get_check_acl_after_first_put "$1" "$BUCKET_ONE_NAME"
8079
assert_success
8180

82-
run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$USERNAME_ONE" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"
81+
run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$username" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"
8382
assert_success
8483

8584
run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file"

‎tests/test_rest.sh

+1-4
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,7 @@ test_file="test_file"
375375

376376
@test "REST - head object" {
377377
if [ "$DIRECT" != "true" ]; then
378-
skip "https://github.com/versity/versitygw/issues/1018"
378+
skip "https://github.com/versity/versitygw/issues/1114"
379379
fi
380380
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
381381
assert_success
@@ -449,9 +449,6 @@ test_file="test_file"
449449
}
450450

451451
@test "REST - HeadObject does not return 405 with versioning, after file deleted" {
452-
if [ "$DIRECT" != "true" ]; then
453-
skip "https://github.com/versity/versitygw/issues/1029"
454-
fi
455452
if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
456453
skip
457454
fi

‎tests/test_rest_checksum.sh

+72-15
Original file line numberDiff line numberDiff line change
@@ -23,34 +23,59 @@ source ./tests/util/util_setup.sh
2323

2424
test_file="test_file"
2525

26-
@test "REST - HeadObject returns x-amz-checksum-sha256" {
26+
@test "REST - invalid checksum type" {
27+
if [ "$DIRECT" != "true" ]; then
28+
skip "https://github.com/versity/versitygw/issues/1104"
29+
fi
2730
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
2831
assert_success
2932

30-
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256"
33+
run check_invalid_checksum_type "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
3134
assert_success
35+
}
3236

33-
run check_checksum_rest_sha256 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
37+
@test "REST - sha256 checksum - invalid" {
38+
run check_checksum_rest_invalid "sha256"
3439
assert_success
3540
}
3641

37-
@test "REST - PutObject rejects invalid sha256 checksum" {
38-
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
42+
@test "REST - sha256 checksum - incorrect" {
43+
run check_checksum_rest_incorrect "sha256"
3944
assert_success
45+
}
4046

41-
run put_object_rest_sha256_invalid "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
47+
@test "REST - sha256 checksum - correct" {
48+
run add_correct_checksum "sha256"
4249
assert_success
4350
}
4451

45-
@test "REST - PutObject rejects incorrect sha256 checksum" {
52+
@test "REST - HeadObject returns x-amz-checksum-sha256" {
4653
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
4754
assert_success
4855

49-
run put_object_rest_sha256_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
56+
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256"
57+
assert_success
58+
59+
run check_checksum_rest_sha256 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
60+
assert_success
61+
}
62+
63+
@test "REST - crc32 checksum - invalid" {
64+
run check_checksum_rest_invalid "crc32c"
65+
assert_success
66+
}
67+
68+
@test "REST - crc32 checksum - incorrect" {
69+
run check_checksum_rest_incorrect "crc32"
5070
assert_success
5171
}
5272

5373
@test "REST - crc32 checksum - correct" {
74+
run add_correct_checksum "crc32"
75+
assert_success
76+
}
77+
78+
@test "REST - crc32 checksum - HeadObject" {
5479
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
5580
assert_success
5681

@@ -61,26 +86,58 @@ test_file="test_file"
6186
assert_success
6287
}
6388

64-
@test "REST - crc32 checksum - incorrect" {
65-
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
89+
@test "REST - crc64nvme checksum - invalid" {
90+
run check_checksum_rest_invalid "crc64nvme"
6691
assert_success
92+
}
6793

68-
run put_object_rest_crc32_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
94+
@test "REST - crc64nvme checksum - incorrect" {
95+
run check_checksum_rest_incorrect "crc64nvme"
6996
assert_success
7097
}
7198

7299
@test "REST - crc64nvme checksum - correct" {
73-
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
100+
run add_correct_checksum "sha256"
74101
assert_success
102+
}
75103

76-
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc64nvme"
104+
@test "REST - crc32c checksum - invalid" {
105+
run check_checksum_rest_invalid "crc32c"
77106
assert_success
78107
}
79108

80-
@test "REST - crc64nvme checksum - incorrect" {
109+
@test "REST - crc32c checksum - incorrect" {
110+
run check_checksum_rest_incorrect "crc32c"
111+
assert_success
112+
}
113+
114+
@test "REST - crc32c checksum - correct" {
115+
run add_correct_checksum "crc32c"
116+
assert_success
117+
}
118+
119+
@test "REST - sha1 checksum - invalid" {
120+
run check_checksum_rest_invalid "sha1"
121+
assert_success
122+
}
123+
124+
@test "REST - sha1 checksum - incorrect" {
125+
run check_checksum_rest_incorrect "sha1"
126+
assert_success
127+
}
128+
129+
@test "REST - sha1 checksum - correct" {
130+
run add_correct_checksum "sha1"
131+
assert_success
132+
}
133+
134+
@test "REST - attempt to get checksum without checksum mode" {
81135
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
82136
assert_success
83137

84-
run put_object_rest_crc64nvme_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
138+
run add_correct_checksum "sha256"
139+
assert_success
140+
141+
run head_object_without_and_with_checksum "$BUCKET_ONE_NAME" "$test_file"
85142
assert_success
86143
}

‎tests/test_rest_chunked.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ source ./tests/util/util_setup.sh
3838

3939
@test "REST - chunked upload, signature error" {
4040
if [ "$DIRECT" != "true" ]; then
41-
skip "https://github.com/versity/versitygw/issues/1056 - gibberish at end"
41+
skip "https://github.com/versity/versitygw/issues/1123"
4242
fi
4343
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
4444
assert_success

0 commit comments

Comments
 (0)