Merge pull request #677 from versity/test_cmdline_more_user_ops

Test cmdline more user ops

Author: benmcclelland

.github/workflows/system.yml

@@ -8,17 +8,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          #- set: 1
-          #  LOCAL_FOLDER: /tmp/gw1
-          #  BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
-          #  BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
-          #  IAM_TYPE: folder
-          #  USERS_FOLDER: /tmp/iam1
-          #  AWS_ENDPOINT_URL: https://127.0.0.1:7070
-          #  RUN_SET: "s3cmd"
-          #  RECREATE_BUCKETS: "true"
-          #  PORT: 7070
-          #  BACKEND: "posix"
+          - set: 1
+            LOCAL_FOLDER: /tmp/gw1
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam1
+            AWS_ENDPOINT_URL: https://127.0.0.1:7070
+            RUN_SET: "s3cmd"
+            RECREATE_BUCKETS: "true"
+            PORT: 7070
+            BACKEND: "posix"
           - set: 2
             LOCAL_FOLDER: /tmp/gw2
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-2

tests/commands/abort_multipart_upload.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 abort_multipart_upload() {
+  record_command "abort-multipart-upload" "client:s3api"
   if [ $# -ne 3 ]; then
     log 2 "'abort multipart upload' command requires bucket, key, upload ID"
     return 1
@@ -13,6 +14,7 @@ abort_multipart_upload() {
 }
 
 abort_multipart_upload_with_user() {
+  record_command "abort-multipart-upload" "client:s3api"
   if [ $# -ne 5 ]; then
     log 2 "'abort multipart upload' command requires bucket, key, upload ID, username, password"
     return 1

tests/commands/complete_multipart_upload.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 complete_multipart_upload() {
+  record_command "complete-multipart-upload" "client:s3api"
   if [[ $# -ne 4 ]]; then
     log 2 "'complete multipart upload' command requires bucket, key, upload ID, parts list"
     return 1

tests/commands/copy_object.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 copy_object() {
+  record-command "copy-object" "client:$1"
   if [ $# -ne 4 ]; then
     echo "copy object command requires command type, source, bucket, key"
     return 1
@@ -29,6 +30,7 @@ copy_object() {
 }
 
 copy_object_empty() {
+  record-command "copy-object" "client:s3api"
   error=$(aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
   if [[ $result -eq 0 ]]; then
     log 2 "copy object with empty parameters returned no error"

tests/commands/create_bucket.sh

@@ -1,9 +1,12 @@
 #!/usr/bin/env bash
 
+source ./tests/report.sh
+
 # create an AWS bucket
 # param:  bucket name
 # return 0 for success, 1 for failure
 create_bucket() {
+  record_command "create-bucket" "client:$1"
   if [ $# -ne 2 ]; then
     log 2 "create bucket missing command type, bucket name"
     return 1
@@ -33,6 +36,7 @@ create_bucket() {
 }
 
 create_bucket_object_lock_enabled() {
+  record_command "create-bucket" "client:s3api"
   if [ $# -ne 1 ]; then
     log 2 "create bucket missing bucket name"
     return 1

tests/commands/create_multipart_upload.sh

@@ -4,6 +4,7 @@
 # params:  bucket, key
 # return 0 for success, 1 for failure
 create_multipart_upload() {
+  record_command "create-multipart-upload" "client:s3api"
   if [ $# -ne 2 ]; then
     log 2 "create multipart upload function must have bucket, key"
     return 1
@@ -24,6 +25,7 @@ create_multipart_upload() {
 }
 
 create_multipart_upload_with_user() {
+  record_command "create-multipart-upload" "client:s3api"
   if [ $# -ne 4 ]; then
     log 2 "create multipart upload function must have bucket, key, username, password"
     return 1
@@ -44,6 +46,7 @@ create_multipart_upload_with_user() {
 }
 
 create_multipart_upload_params() {
+  record_command "create-multipart-upload" "client:s3api"
   if [ $# -ne 8 ]; then
     log 2 "create multipart upload function with params must have bucket, key, content type, metadata, object lock legal hold status, " \
       "object lock mode, object lock retain until date, and tagging"
@@ -71,6 +74,7 @@ create_multipart_upload_params() {
 }
 
 create_multipart_upload_custom() {
+  record_command "create-multipart-upload" "client:s3api"
   if [ $# -lt 2 ]; then
     log 2 "create multipart upload custom function must have at least bucket and key"
     return 1

tests/commands/delete_bucket.sh

@@ -4,11 +4,17 @@
 # param:  bucket name
 # return 0 for success, 1 for failure
 delete_bucket() {
+  record_command "delete-bucket" "client:$1"
   if [ $# -ne 2 ]; then
     log 2 "delete bucket missing command type, bucket name"
     return 1
   fi
 
+  if [[ ( $RECREATE_BUCKETS == "false" ) && (( "$2" == "$BUCKET_ONE_NAME" ) || ( "$2" == "$BUCKET_TWO_NAME" )) ]]; then
+    log 2 "attempt to delete main buckets in static mode"
+    return 1
+  fi
+
   local exit_code=0
   local error
   if [[ $1 == 's3' ]]; then

tests/commands/delete_bucket_policy.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 delete_bucket_policy() {
+  record_command "delete-bucket-policy" "client:$1"
   if [[ $# -ne 2 ]]; then
     log 2 "delete bucket policy command requires command type, bucket"
     return 1
@@ -23,6 +24,7 @@ delete_bucket_policy() {
 }
 
 delete_bucket_policy_with_user() {
+  record_command "delete-bucket-policy" "client:s3api"
   if [[ $# -ne 3 ]]; then
     log 2 "'delete bucket policy with user' command requires bucket, username, password"
     return 1

tests/commands/delete_bucket_tagging.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+delete_bucket_tagging() {
+  record_command "delete-bucket-tagging" "client:$1"
+  if [ $# -ne 2 ]; then
+    log 2 "delete bucket tagging command missing command type, bucket name"
+    return 1
+  fi
+  local result
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
+  else
+    log 2 "invalid command type $1"
+    return 1
+  fi
+  if [[ $result -ne 0 ]]; then
+    log 2 "error deleting bucket tagging: $tags"
+    return 1
+  fi
+  return 0
+}

tests/commands/delete_object.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 delete_object() {
+  record_command "delete-object" "client:$1"
   if [ $# -ne 3 ]; then
     log 2 "delete object command requires command type, bucket, key"
     return 1
@@ -28,6 +29,7 @@ delete_object() {
 }
 
 delete_object_with_user() {
+  record_command "delete-object" "client:$1"
   if [ $# -ne 5 ]; then
     log 2 "delete object with user command requires command type, bucket, key, access ID, secret key"
     return 1

tests/commands/delete_object_tagging.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 delete_object_tagging() {
+  record_command "delete-object-tagging" "client:$1"
   if [[ $# -ne 3 ]]; then
     echo "delete object tagging command missing command type, bucket, key"
     return 1

tests/commands/delete_objects.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+delete_objects() {
+  record_command "delete-objects" "client:s3api"
+  if [[ $# -ne 3 ]]; then
+    log 2 "'delete-objects' command requires bucket name, two object keys"
+    return 1
+  fi
+  if ! error=$(aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{
+      \"Objects\": [
+        {\"Key\": \"$2\"},
+        {\"Key\": \"$3\"}
+      ]
+    }" 2>&1); then
+    log 2 "error deleting objects: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/get_bucket_acl.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 get_bucket_acl() {
+  record_command "get-bucket-acl" "client:$1"
   if [ $# -ne 2 ]; then
     log 2 "bucket ACL command missing command type, bucket name"
     return 1
@@ -22,6 +23,7 @@ get_bucket_acl() {
 }
 
 get_bucket_acl_with_user() {
+  record_command "get-bucket-acl" "client:s3api"
   if [ $# -ne 3 ]; then
     log 2 "'get bucket ACL with user' command requires bucket name, username, password"
     return 1

tests/commands/get_bucket_location.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 get_bucket_location() {
+  record_command "get-bucket-location" "client:$1"
   if [[ $# -ne 2 ]]; then
     echo "get bucket location command requires command type, bucket name"
     return 1
@@ -23,6 +24,7 @@ get_bucket_location() {
 }
 
 get_bucket_location_aws() {
+  record_command "get-bucket-location" "client:s3api"
   if [[ $# -ne 1 ]]; then
     echo "get bucket location (aws) requires bucket name"
     return 1
@@ -38,6 +40,7 @@ get_bucket_location_aws() {
 }
 
 get_bucket_location_s3cmd() {
+  record_command "get-bucket-location" "client:s3cmd"
   if [[ $# -ne 1 ]]; then
     echo "get bucket location (s3cmd) requires bucket name"
     return 1
@@ -53,6 +56,7 @@ get_bucket_location_s3cmd() {
 }
 
 get_bucket_location_mc() {
+  record_command "get-bucket-location" "client:mc"
   if [[ $# -ne 1 ]]; then
     echo "get bucket location (mc) requires bucket name"
     return 1

tests/commands/get_bucket_ownership_controls.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 get_bucket_ownership_controls() {
+  record_command "get-bucket-ownership-controls" "client:s3api"
   if [[ $# -ne 1 ]]; then
     log 2 "'get bucket ownership controls' command requires bucket name"
     return 1

tests/commands/get_bucket_policy.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 get_bucket_policy() {
+  record_command "get-bucket-policy" "client:$1"
   if [[ $# -ne 2 ]]; then
     log 2 "get bucket policy command requires command type, bucket"
     return 1
@@ -25,6 +26,7 @@ get_bucket_policy() {
 }
 
 get_bucket_policy_aws() {
+  record_command "get-bucket-policy" "client:s3api"
   if [[ $# -ne 1 ]]; then
     log 2 "aws 'get bucket policy' command requires bucket"
     return 1
@@ -47,6 +49,7 @@ get_bucket_policy_aws() {
 }
 
 get_bucket_policy_with_user() {
+  record_command "get-bucket-policy" "client:s3api"
   if [[ $# -ne 3 ]]; then
     log 2 "'get bucket policy with user' command requires bucket, username, password"
     return 1
@@ -67,6 +70,7 @@ get_bucket_policy_with_user() {
 }
 
 get_bucket_policy_s3cmd() {
+  record_command "get-bucket-policy" "client:s3cmd"
   if [[ $# -ne 1 ]]; then
     log 2 "s3cmd 'get bucket policy' command requires bucket"
     return 1
@@ -110,6 +114,7 @@ get_bucket_policy_s3cmd() {
 }
 
 get_bucket_policy_mc() {
+  record_command "get-bucket-policy" "client:mc"
   if [[ $# -ne 1 ]]; then
     echo "aws 'get bucket policy' command requires bucket"
     return 1

tests/commands/get_bucket_tagging.sh

@@ -4,6 +4,7 @@
 # params:  bucket
 # export 'tags' on success, return 1 for error
 get_bucket_tagging() {
+  record_command "get-bucket-tagging" "client:$1"
   if [ $# -ne 2 ]; then
     echo "get bucket tag command missing command type, bucket name"
     return 1

tests/commands/get_bucket_versioning.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 get_bucket_versioning() {
+  record_command "get-bucket-versioning" "client:s3api"
   if [[ $# -ne 2 ]]; then
     log 2 "put bucket versioning command requires command type, bucket name"
     return 1

tests/commands/get_object.sh

@@ -1,33 +1,36 @@
 #!/usr/bin/env bash
 
 get_object() {
+  record_command "get-object" "client:$1"
   if [ $# -ne 4 ]; then
     log 2 "get object command requires command type, bucket, key, destination"
     return 1
   fi
   local exit_code=0
   local error
   if [[ $1 == 's3' ]]; then
-    error=$(aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+    get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
   else
     log 2 "'get object' command not implemented for '$1'"
     return 1
   fi
   log 5 "get object exit code: $exit_code"
   if [ $exit_code -ne 0 ]; then
-    log 2 "error getting object: $error"
+    log 2 "error getting object: $get_object_error"
+    export get_object_error
     return 1
   fi
   return 0
 }
 
 get_object_with_range() {
+  record_command "get-object" "client:s3api"
   if [[ $# -ne 4 ]]; then
     log 2 "'get object with range' requires bucket, key, range, outfile"
     return 1
@@ -41,6 +44,7 @@ get_object_with_range() {
 }
 
 get_object_with_user() {
+  record_command "get-object" "client:$1"
   if [ $# -ne 6 ]; then
     log 2 "'get object with user' command requires command type, bucket, key, save location, aws ID, aws secret key"
     return 1

tests/commands/get_object_attributes.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 get_object_attributes() {
+  record_command "get-object-attributes" "client:s3api"
   if [[ $# -ne 2 ]]; then
     log 2 "'get object attributes' command requires bucket, key"
     return 1