Skip to content

Commit d10bd4d

Browse files
lrm25lrm25
committed
test: crc64nvme checksum
1 parent 64a96fe commit d10bd4d

8 files changed

+160
-24
lines changed

‎tests/rest_scripts/calculate_crc64nvme.sh

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2024 Versity Software
4+
# This file is licensed under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance
6+
# with the License. You may obtain a copy of the License at
7+
#
8+
# http:#www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing,
11+
# software distributed under the License is distributed on an
12+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
13+
# KIND, either express or implied. See the License for the
14+
# specific language governing permissions and limitations
15+
# under the License.
16+
17+
source ./tests/rest_scripts/rest.sh
18+
19+
if [ -z "$TEST_FILE_FOLDER" ] || [ -z "$DATA_FILE" ]; then
20+
log_rest 2 "TEST_FILE_FOLDER and DATA_FILE params required for crc64nvme checksum (for python virtual env)"
21+
exit 1
22+
fi
23+
if ! python3 -m venv "$TEST_FILE_FOLDER/env" 1>/dev/null; then
24+
log_rest 2 "error creating python virtual environment"
25+
exit 1
26+
fi
27+
if ! source "$TEST_FILE_FOLDER/env/bin/activate" 1>/dev/null; then
28+
log_rest 2 "error activating virtual environment"
29+
exit 1
30+
fi
31+
if ! python3 -m pip install awscrt 1>/dev/null 2>&1; then
32+
log_rest 2 "error installing awscrt"
33+
exit 1
34+
fi
35+
if ! checksum_decimal=$(python3 -c "
36+
import sys
37+
from awscrt import checksums
38+
39+
with open(sys.argv[1], 'rb') as f:
40+
print(checksums.crc64nvme(f.read()))" "$DATA_FILE" 2>&1); then
41+
log_rest 2 "error calculating checksum: $checksum_decimal"
42+
exit 1
43+
fi
44+
log 5 "decimal checksum: $checksum_decimal"
45+
if ! deactivate 1>/dev/null; then
46+
log_rest 2 "error deactivating virtual environment"
47+
exit 1
48+
fi
49+
checksum_hash=$(printf "%016x" "$checksum_decimal" | xxd -r -p | base64)
50+
echo "$checksum_hash"

‎tests/rest_scripts/get_object_lock_config.sh

-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ if ! build_canonical_request "${canonical_request_data[@]}"; then
2828
log_rest 2 "error building request"
2929
exit 1
3030
fi
31-
echo "$canonical_request" > "cr.txt"
3231

3332
# shellcheck disable=SC2119
3433
create_canonical_hash_sts_and_signature

‎tests/rest_scripts/put_object.sh

+8
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,14 @@ elif [ "$checksum_type" == "crc32" ]; then
5151
checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
5252
fi
5353
cr_data+=("x-amz-checksum-crc32:$checksum_hash")
54+
elif [ "$checksum_type" == "crc64nvme" ]; then
55+
if [ -z "$checksum_hash" ]; then
56+
if ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
57+
log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
58+
exit 1
59+
fi
60+
fi
61+
cr_data+=("x-amz-checksum-crc64nvme:$checksum_hash")
5462
fi
5563
cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
5664
build_canonical_request "${cr_data[@]}"

‎tests/test_rest_checksum.sh

+22-4
Original file line numberDiff line numberDiff line change
@@ -51,23 +51,41 @@ source ./tests/util/util_setup.sh
5151
assert_success
5252
}
5353

54-
@test "REST - crc32 checksum - success" {
54+
@test "REST - crc32 checksum - correct" {
5555
test_file="test_file"
5656
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
5757
assert_success
5858

5959
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc32"
6060
assert_success
61+
62+
run check_checksum_rest_crc32 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
63+
assert_success
6164
}
6265

63-
@test "REST - crc32 checksum - correct" {
66+
@test "REST - crc32 checksum - incorrect" {
6467
test_file="test_file"
6568
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
6669
assert_success
6770

68-
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc32"
71+
run put_object_rest_crc32_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
6972
assert_success
73+
}
7074

71-
run check_checksum_rest_crc32 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
75+
@test "REST - crc64nvme checksum - correct" {
76+
test_file="test_file"
77+
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
78+
assert_success
79+
80+
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc64nvme"
81+
assert_success
82+
}
83+
84+
@test "REST - crc64nvme checksum - incorrect" {
85+
test_file="test_file"
86+
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
87+
assert_success
88+
89+
run put_object_rest_crc64nvme_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
7290
assert_success
7391
}

‎tests/util/util_acl.sh

+13-6
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,17 @@ get_check_acl_after_policy() {
216216
fi
217217
}
218218

219+
check_direct_display_name() {
220+
if ! display_name=$(echo "$owner" | xmllint --xpath '//*[local-name()="DisplayName"]/text()' - 2>&1); then
221+
log 2 "error getting display name: $display_name"
222+
return 1
223+
fi
224+
if [ "$display_name" != "$DIRECT_DISPLAY_NAME" ]; then
225+
log 2 "display name mismatch (expected '$DIRECT_DISPLAY_NAME', actual '$display_name')"
226+
return 1
227+
fi
228+
}
229+
219230
get_and_check_acl_rest() {
220231
if [ $# -ne 1 ]; then
221232
log 2 "'get_and_check_acl_rest' requires bucket name"
@@ -239,12 +250,8 @@ get_and_check_acl_rest() {
239250
return 1
240251
fi
241252
if [ "$DIRECT" == "true" ]; then
242-
if ! display_name=$(echo "$owner" | xmllint --xpath '//*[local-name()="DisplayName"]/text()' - 2>&1); then
243-
log 2 "error getting display name: $display_name"
244-
return 1
245-
fi
246-
if [ "$display_name" != "$DIRECT_DISPLAY_NAME" ]; then
247-
log 2 "display name mismatch (expected '$DIRECT_DISPLAY_NAME', actual '$display_name')"
253+
if ! check_direct_display_name; then
254+
log 2 "error checking direct display name"
248255
return 1
249256
fi
250257
else

‎tests/util/util_head_object.sh

+10
Original file line numberDiff line numberDiff line change
@@ -166,3 +166,13 @@ check_checksum_rest_crc32() {
166166
fi
167167
return 0
168168
}
169+
170+
crc64_file() {
171+
source ./env/bin/activate
172+
python3 -m pip install awscrt
173+
if ! checksum=$(python3 -c "import sys;from awscrt import checksums;crc64nvme_value = checksums.crc64nvme(sys.argv[1]);print(crc64nvme_value)" "$1"); then
174+
return 1
175+
fi
176+
log 5 "checksum: $checksum"
177+
deactivate
178+
}

‎tests/util/util_object.sh

+48-8
Original file line numberDiff line numberDiff line change
@@ -297,20 +297,20 @@ list_and_check_directory_obj() {
297297
return 0
298298
}
299299

300-
check_sha256_invalid_or_incorrect() {
301-
if [ $# -ne 5 ]; then
302-
log 2 "'check_sha256_invalid_or_incorrect' requires data file, bucket name, key, checksum, expected error"
300+
check_checksum_invalid_or_incorrect() {
301+
if [ $# -ne 6 ]; then
302+
log 2 "'check_sha256_invalid_or_incorrect' requires data file, bucket name, key, checksum type, checksum, expected error"
303303
return 1
304304
fi
305-
if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="sha256" CHECKSUM="$4" ./tests/rest_scripts/put_object.sh 2>&1); then
305+
if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="$4" CHECKSUM="$5" ./tests/rest_scripts/put_object.sh 2>&1); then
306306
log 2 "error: $result"
307307
return 1
308308
fi
309309
if [ "$result" != "400" ]; then
310310
log 2 "expected response code of '400', was '$result' (response: $(cat "$TEST_FILE_FOLDER/result.txt")"
311311
return 1
312312
fi
313-
if ! check_xml_element "$TEST_FILE_FOLDER/result.txt" "$5" "Error" "Message"; then
313+
if ! check_xml_element "$TEST_FILE_FOLDER/result.txt" "$6" "Error" "Message"; then
314314
log 2 "xml error message mismatch"
315315
return 1
316316
fi
@@ -322,7 +322,8 @@ put_object_rest_checksum() {
322322
log 2 "'put_object_rest_sha256_checksum' requires data file, bucket name, key, checksum type"
323323
return 1
324324
fi
325-
if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="$4" ./tests/rest_scripts/put_object.sh 2>&1); then
325+
# shellcheck disable=SC2097,SC2098
326+
if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="$4" ./tests/rest_scripts/put_object.sh 2>&1); then
326327
log 2 "error: $result"
327328
return 1
328329
fi
@@ -339,7 +340,7 @@ put_object_rest_sha256_invalid() {
339340
log 2 "'put_object_rest_sha256_invalid' requires data file, bucket name, key"
340341
return 1
341342
fi
342-
if ! check_sha256_invalid_or_incorrect "$1" "$2" "$3" "dummy" "Value for x-amz-checksum-sha256 header is invalid."; then
343+
if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256" "dummy" "Value for x-amz-checksum-sha256 header is invalid."; then
343344
log 2 "error checking checksum"
344345
return 1
345346
fi
@@ -357,7 +358,7 @@ put_object_rest_sha256_incorrect() {
357358
error_message="The sha256 you specified did not match the calculated checksum."
358359
fi
359360
incorrect_checksum="$(echo -n "dummy" | sha256sum | awk '{print $1}' | xxd -r -p | base64)"
360-
if ! check_sha256_invalid_or_incorrect "$1" "$2" "$3" "$incorrect_checksum" "$error_message"; then
361+
if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256" "$incorrect_checksum" "$error_message"; then
361362
log 2 "error checking checksum"
362363
return 1
363364
fi
@@ -379,3 +380,42 @@ put_object_rest_chunked_payload_type_without_content_length() {
379380
fi
380381
return 0
381382
}
383+
384+
put_object_rest_crc32_incorrect() {
385+
if [ $# -ne 3 ]; then
386+
log 2 "'put_object_rest_crc32_incorrect' requires data file, bucket name, key"
387+
return 1
388+
fi
389+
if [ "$DIRECT" == "true" ]; then
390+
error_message="The CRC32 you specified did not match the calculated checksum."
391+
else
392+
error_message="The crc32 you specified did not match the calculated checksum."
393+
fi
394+
incorrect_checksum="$(echo -n "dummy" | gzip -c -1 | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
395+
if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "crc32" "$incorrect_checksum" "$error_message"; then
396+
log 2 "error checking checksum"
397+
return 1
398+
fi
399+
return 0
400+
}
401+
402+
put_object_rest_crc64nvme_incorrect() {
403+
if [ $# -ne 3 ]; then
404+
log 2 "'put_object_rest_crc64nvme_incorrect' requires data file, bucket name, key"
405+
return 1
406+
fi
407+
if [ "$DIRECT" == "true" ]; then
408+
error_message="The CRC64NVME you specified did not match the calculated checksum."
409+
else
410+
error_message="The crc64nvme you specified did not match the calculated checksum."
411+
fi
412+
if ! incorrect_checksum=$(DATA_FILE=<(echo -n "dummy") TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
413+
log 2 "error calculating checksum: $incorrect_checksum"
414+
return 1
415+
fi
416+
if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "crc64nvme" "$incorrect_checksum" "$error_message"; then
417+
log 2 "error checking checksum"
418+
return 1
419+
fi
420+
return 0
421+
}

‎tests/util/util_policy.sh

+9-5
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,14 @@ check_for_empty_policy() {
4242
return 0
4343
}
4444

45+
add_direct_user_to_principal() {
46+
if [ "${principals[$idx]}" == "*" ]; then
47+
modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/$DIRECT_S3_ROOT_ACCOUNT_NAME\"}"
48+
else
49+
modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/${principals[$idx]}\"}"
50+
fi
51+
}
52+
4553
get_modified_principal() {
4654
log 6 "get_modified_principal"
4755
if [ $# -ne 1 ]; then
@@ -55,11 +63,7 @@ get_modified_principal() {
5563
fi
5664
for ((idx=0; idx<${#principals[@]}; idx++)); do
5765
if [ "$DIRECT" == "true" ]; then
58-
if [ "${principals[$idx]}" == "*" ]; then
59-
modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/$DIRECT_S3_ROOT_ACCOUNT_NAME\"}"
60-
else
61-
modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/${principals[$idx]}\"}"
62-
fi
66+
add_direct_user_to_principal
6367
else
6468
# shellcheck disable=SC2089
6569
modified_principal+="\"${principals[$idx]}\""

0 commit comments

Comments
 (0)