Skip to content

Commit 2232fb8

Browse files
lrm25lrm25
committed
test: tagging, more run/assert changes, dockerfile, test split-ups
1 parent 9f9f895 commit 2232fb8

15 files changed

+698
-449
lines changed

‎.github/workflows/docker-bats.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,4 @@ jobs:
2525
run: sudo apt-get install -y docker-compose
2626

2727
- name: Run Docker Container
28-
run: docker-compose -f tests/docker-compose-bats.yml up --exit-code-from s3api_only s3api_only
28+
run: docker-compose -f tests/docker-compose-bats.yml up --exit-code-from s3api_np_only s3api_np_only

‎.github/workflows/system.yml

+25-3
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030
RECREATE_BUCKETS: "true"
3131
PORT: 7071
3232
BACKEND: "posix"
33-
- set: "s3api, posix"
33+
- set: "s3api non-policy, posix"
3434
LOCAL_FOLDER: /tmp/gw3
3535
BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
3636
BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
@@ -74,7 +74,7 @@ jobs:
7474
RECREATE_BUCKETS: "false"
7575
PORT: 7075
7676
BACKEND: "posix"
77-
- set: "s3api, s3 backend"
77+
- set: "s3api non-policy, s3 backend"
7878
LOCAL_FOLDER: /tmp/gw7
7979
BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
8080
BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
@@ -118,6 +118,28 @@ jobs:
118118
RECREATE_BUCKETS: "false"
119119
PORT: 7079
120120
BACKEND: "posix"
121+
- set: "s3api policy and user, posix"
122+
LOCAL_FOLDER: /tmp/gw11
123+
BUCKET_ONE_NAME: versity-gwtest-bucket-one-10
124+
BUCKET_TWO_NAME: versity-gwtest-bucket-two-10
125+
IAM_TYPE: folder
126+
USERS_FOLDER: /tmp/iam11
127+
AWS_ENDPOINT_URL: https://127.0.0.1:7080
128+
RUN_SET: "s3api-policy,s3api-user"
129+
RECREATE_BUCKETS: "true"
130+
PORT: 7080
131+
BACKEND: "posix"
132+
- set: "s3api policy and user, s3 backend"
133+
LOCAL_FOLDER: /tmp/gw12
134+
BUCKET_ONE_NAME: versity-gwtest-bucket-one-11
135+
BUCKET_TWO_NAME: versity-gwtest-bucket-two-11
136+
IAM_TYPE: folder
137+
USERS_FOLDER: /tmp/iam12
138+
AWS_ENDPOINT_URL: https://127.0.0.1:7081
139+
RUN_SET: "s3api-policy,s3api-user"
140+
RECREATE_BUCKETS: "true"
141+
PORT: 7081
142+
BACKEND: "s3"
121143
steps:
122144
- name: Check out code into the Go module directory
123145
uses: actions/checkout@v4
@@ -152,7 +174,7 @@ jobs:
152174
run: |
153175
sudo apt-get install libxml2-utils
154176
155-
- name: Build and run, posix backend
177+
- name: Build and run
156178
env:
157179
LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
158180
BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}

‎tests/commands/delete_object_tagging.sh

+46-1
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,13 @@ delete_object_tagging() {
2020
echo "delete object tagging command missing command type, bucket, key"
2121
return 1
2222
fi
23+
delete_result=0
2324
if [[ $1 == 'aws' ]]; then
2425
error=$(aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$?
2526
elif [[ $1 == 'mc' ]]; then
2627
error=$(mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$?
28+
elif [ "$1" == 'rest' ]; then
29+
delete_object_tagging_rest "$2" "$3" || delete_result=$?
2730
else
2831
echo "delete-object-tagging command not implemented for '$1'"
2932
return 1
@@ -33,4 +36,46 @@ delete_object_tagging() {
3336
return 1
3437
fi
3538
return 0
36-
}
39+
}
40+
41+
delete_object_tagging_rest() {
42+
if [ $# -ne 2 ]; then
43+
log 2 "'delete_object_tagging' requires bucket, key"
44+
return 1
45+
fi
46+
47+
generate_hash_for_payload ""
48+
49+
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
50+
aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
51+
header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
52+
# shellcheck disable=SC2154
53+
canonical_request="DELETE
54+
/$1/$2
55+
tagging=
56+
host:$aws_endpoint_url_address
57+
x-amz-content-sha256:$payload_hash
58+
x-amz-date:$current_date_time
59+
60+
host;x-amz-content-sha256;x-amz-date
61+
$payload_hash"
62+
63+
if ! generate_sts_string "$current_date_time" "$canonical_request"; then
64+
log 2 "error generating sts string"
65+
return 1
66+
fi
67+
get_signature
68+
# shellcheck disable=SC2154
69+
reply=$(curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2?tagging" \
70+
-H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
71+
-H "x-amz-content-sha256: $payload_hash" \
72+
-H "x-amz-date: $current_date_time" \
73+
-d "$tagging" -o "$TEST_FILE_FOLDER"/delete_tagging_error.txt 2>&1)
74+
log 5 "reply status code: $reply"
75+
if [[ "$reply" != "204" ]]; then
76+
log 2 "reply error: $reply"
77+
log 2 "put object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/delete_tagging_error.txt)"
78+
return 1
79+
fi
80+
return 0
81+
}

‎tests/commands/get_object.sh

+1-2
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,7 @@ get_object_with_range() {
5050
log 2 "'get object with range' requires bucket, key, range, outfile"
5151
return 1
5252
fi
53-
get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1) || local exit_code=$?
54-
if [[ $exit_code -ne 0 ]]; then
53+
if ! get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1); then
5554
log 2 "error getting object with range: $get_object_error"
5655
return 1
5756
fi

‎tests/commands/get_object_tagging.sh

+50-2
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@ get_object_tagging() {
2121
return 1
2222
fi
2323
local result
24-
if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
24+
if [[ "$1" == 'aws' ]] || [[ $1 == 's3api' ]]; then
2525
tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
26-
elif [[ $1 == 'mc' ]]; then
26+
elif [[ "$1" == 'mc' ]]; then
2727
tags=$(mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?
28+
elif [ "$1" == 'rest' ]; then
29+
get_object_tagging_rest "$2" "$3" || result=$?
2830
else
2931
log 2 "invalid command type $1"
3032
return 1
@@ -41,4 +43,50 @@ get_object_tagging() {
4143
tags=$(echo "$tags" | grep -v "InsecureRequestWarning")
4244
fi
4345
export tags
46+
}
47+
48+
get_object_tagging_rest() {
49+
if [ $# -ne 2 ]; then
50+
log 2 "'get_object_tagging' requires bucket, key"
51+
return 1
52+
fi
53+
54+
generate_hash_for_payload ""
55+
56+
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
57+
aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
58+
header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
59+
# shellcheck disable=SC2154
60+
canonical_request="GET
61+
/$1/$2
62+
tagging=
63+
host:$aws_endpoint_url_address
64+
x-amz-content-sha256:$payload_hash
65+
x-amz-date:$current_date_time
66+
67+
host;x-amz-content-sha256;x-amz-date
68+
$payload_hash"
69+
70+
if ! generate_sts_string "$current_date_time" "$canonical_request"; then
71+
log 2 "error generating sts string"
72+
return 1
73+
fi
74+
get_signature
75+
# shellcheck disable=SC2154
76+
reply=$(curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?tagging" \
77+
-H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
78+
-H "x-amz-content-sha256: $payload_hash" \
79+
-H "x-amz-date: $current_date_time" \
80+
-o "$TEST_FILE_FOLDER"/object_tags.txt 2>&1)
81+
log 5 "reply status code: $reply"
82+
if [[ "$reply" != "200" ]]; then
83+
if [ "$reply" == "404" ]; then
84+
return 1
85+
fi
86+
log 2 "reply error: $reply"
87+
log 2 "get object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/object_tags.txt)"
88+
return 2
89+
fi
90+
log 5 "object tags: $(cat "$TEST_FILE_FOLDER"/object_tags.txt)"
91+
return 0
4492
}

‎tests/commands/put_object_tagging.sh

+56-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
put_object_tagging() {
1818
if [ $# -ne 5 ]; then
19-
log 2 "'put-object-tagging' command missing command type, object name, file, key, and/or value"
19+
log 2 "'put-object-tagging' command missing command type, bucket, object name, file, key, and/or value"
2020
return 1
2121
fi
2222
local error
@@ -26,6 +26,8 @@ put_object_tagging() {
2626
error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$?
2727
elif [[ $1 == 'mc' ]]; then
2828
error=$(mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$?
29+
elif [[ $1 == 'rest' ]]; then
30+
put_object_tagging_rest "$2" "$3" "$4" "$5" || result=$?
2931
else
3032
log 2 "invalid command type $1"
3133
return 1
@@ -35,4 +37,56 @@ put_object_tagging() {
3537
return 1
3638
fi
3739
return 0
38-
}
40+
}
41+
42+
put_object_tagging_rest() {
43+
if [ $# -ne 4 ]; then
44+
log 2 "'put_object_tagging' requires bucket, key, tag key, tag value"
45+
return 1
46+
fi
47+
48+
tagging="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
49+
<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
50+
<TagSet>
51+
<Tag>
52+
<Key>$3</Key>
53+
<Value>$4</Value>
54+
</Tag>
55+
</TagSet>
56+
</Tagging>"
57+
58+
generate_hash_for_payload "$tagging"
59+
60+
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
61+
aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
62+
header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
63+
# shellcheck disable=SC2154
64+
canonical_request="PUT
65+
/$1/$2
66+
tagging=
67+
host:$aws_endpoint_url_address
68+
x-amz-content-sha256:$payload_hash
69+
x-amz-date:$current_date_time
70+
71+
host;x-amz-content-sha256;x-amz-date
72+
$payload_hash"
73+
74+
if ! generate_sts_string "$current_date_time" "$canonical_request"; then
75+
log 2 "error generating sts string"
76+
return 1
77+
fi
78+
get_signature
79+
# shellcheck disable=SC2154
80+
reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?tagging" \
81+
-H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
82+
-H "x-amz-content-sha256: $payload_hash" \
83+
-H "x-amz-date: $current_date_time" \
84+
-d "$tagging" -o "$TEST_FILE_FOLDER"/put_tagging_error.txt 2>&1)
85+
log 5 "reply status code: $reply"
86+
if [[ "$reply" != "200" ]]; then
87+
log 2 "reply error: $reply"
88+
log 2 "put object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/put_tagging_error.txt)"
89+
return 1
90+
fi
91+
return 0
92+
}

‎tests/docker-compose-bats.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,14 @@ services:
2525
args:
2626
- CONFIG_FILE=tests/.env.s3
2727
- SECRETS_FILE=tests/.secrets.s3
28-
s3api_only:
28+
s3api_np_only:
2929
build:
3030
context: ../
3131
dockerfile: tests/Dockerfile_test_bats
3232
args:
3333
- CONFIG_FILE=tests/.env.default
3434
image: bats_test
35-
command: ["s3api"]
35+
command: ["s3api-non-policy"]
3636
direct:
3737
build:
3838
context: ../

‎tests/logger.sh

+24-13
Original file line numberDiff line numberDiff line change
@@ -49,29 +49,40 @@ log_mask() {
4949
echo "mask and log requires level, string"
5050
return 1
5151
fi
52-
local masked_args=() # Initialize an array to hold the masked arguments
52+
masked_args=() # Initialize an array to hold the masked arguments
5353

5454
IFS=' ' read -r -a array <<< "$2"
5555

5656
mask_next=false
5757
for arg in "${array[@]}"; do
58-
if [[ $mask_next == true ]]; then
59-
masked_args+=("********")
60-
mask_next=false
61-
elif [[ "$arg" == --secret_key=* ]]; then
62-
masked_args+=("--secret_key=********")
63-
elif [[ "$arg" == --secret=* ]]; then
64-
masked_args+=("--secret=********")
65-
else
66-
if [[ "$arg" == "--secret_key" ]] || [[ "$arg" == "--secret" ]] || [[ "$arg" == "--s3-iam-secret" ]]; then
67-
mask_next=true
68-
fi
69-
masked_args+=("$arg")
58+
if ! check_arg_for_mask "$arg"; then
59+
echo "error checking arg for mask"
60+
return 1
7061
fi
7162
done
7263
log_message "$log_level" "${masked_args[*]}"
7364
}
7465

66+
check_arg_for_mask() {
67+
if [ $# -ne 1 ]; then
68+
echo "'check_arg_for_mask' requires arg"
69+
return 1
70+
fi
71+
if [[ $mask_next == true ]]; then
72+
masked_args+=("********")
73+
mask_next=false
74+
elif [[ "$arg" == --secret_key=* ]]; then
75+
masked_args+=("--secret_key=********")
76+
elif [[ "$arg" == --secret=* ]]; then
77+
masked_args+=("--secret=********")
78+
else
79+
if [[ "$arg" == "--secret_key" ]] || [[ "$arg" == "--secret" ]] || [[ "$arg" == "--s3-iam-secret" ]]; then
80+
mask_next=true
81+
fi
82+
masked_args+=("$arg")
83+
fi
84+
}
85+
7586
log_message() {
7687
if [[ $# -ne 2 ]]; then
7788
echo "log message requires level, message"

0 commit comments

Comments
 (0)