test: versioning, acls work, more cleanup

.github/workflows/system.yml

@@ -8,7 +8,7 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
 
-      - name: Install ShellCheck
+      - name: Install ShellCheck and md5
         run: sudo apt-get install shellcheck
 
       - name: Run ShellCheck

tests/commands/copy_object.sh

@@ -14,7 +14,7 @@ copy_object() {
   elif [[ $1 == 's3cmd' ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure cp "$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
+    error=$(mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
   else
     echo "'copy-object' not implemented for '$1'"
     return 1

tests/commands/delete_bucket.sh

@@ -5,7 +5,7 @@
 # return 0 for success, 1 for failure
 delete_bucket() {
   if [ $# -ne 2 ]; then
-    echo "delete bucket missing command type, bucket name"
+    log 2 "delete bucket missing command type, bucket name"
     return 1
   fi
 
@@ -20,14 +20,14 @@ delete_bucket() {
   elif [[ $1 == 'mc' ]]; then
     error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
   else
-    echo "Invalid command type $1"
+    log 2 "Invalid command type $1"
     return 1
   fi
   if [ $exit_code -ne 0 ]; then
     if [[ "$error" == *"The specified bucket does not exist"* ]]; then
       return 0
     else
-      echo "error deleting bucket: $error"
+      log 2 "error deleting bucket: $error"
       return 1
     fi
   fi

tests/commands/delete_bucket_policy.sh

@@ -2,7 +2,7 @@
 
 delete_bucket_policy() {
   if [[ $# -ne 2 ]]; then
-    echo "delete bucket policy command requires command type, bucket"
+    log 2 "delete bucket policy command requires command type, bucket"
     return 1
   fi
   if [[ $1 == 'aws' ]]; then
@@ -12,11 +12,11 @@ delete_bucket_policy() {
   elif [[ $1 == 'mc' ]]; then
     error=$(mc --insecure anonymous set none "$MC_ALIAS/$2") || delete_result=$?
   else
-    echo "command 'get bucket policy' not implemented for '$1'"
+    log 2 "command 'get bucket policy' not implemented for '$1'"
     return 1
   fi
   if [[ $delete_result -ne 0 ]]; then
-    echo "error deleting bucket policy: $error"
+    log 2 "error deleting bucket policy: $error"
     return 1
   fi
   return 0

tests/commands/get_bucket_acl.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+get_bucket_acl() {
+  if [ $# -ne 2 ]; then
+    log 2 "bucket ACL command missing command type, bucket name"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    acl=$(aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?"
+  elif [[ $1 == 's3cmd' ]]; then
+    acl=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?"
+  else
+    log 2 "command 'get bucket acl' not implemented for $1"
+    return 1
+  fi
+  if [ $exit_code -ne 0 ]; then
+    log 2 "Error getting bucket ACLs: $acl"
+    return 1
+  fi
+  export acl
+}

tests/commands/get_bucket_versioning.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+get_bucket_versioning() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "put bucket versioning command requires command type, bucket name"
+    return 1
+  fi
+  local get_result=0
+  if [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
+  fi
+  if [[ $get_result -ne 0 ]]; then
+    log 2 "error getting bucket versioning: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/list_objects.sh

@@ -58,8 +58,13 @@ list_objects_s3api() {
   done <<< "$output"
 
   object_array=()
-  keys=$(jq -r '.Contents[].Key' <<<"$modified_output")
-  IFS=$'\n' read -rd '' -a object_array <<<"$keys"
+  log 5 "modified output: $modified_output"
+  if echo "$modified_output" | jq -e 'has("Contents")'; then
+    contents=$(echo "$modified_output" | jq -r '.Contents[]')
+    log 5 "contents: $contents"
+    keys=$(echo "$contents" | jq -r '.Key')
+    IFS=$'\n' read -rd '' -a object_array <<<"$keys"
+  fi
 
   export object_array
 }

tests/commands/put_bucket_acl.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+put_bucket_acl() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "put bucket acl command requires command type, bucket name, acls"
+    return 1
+  fi
+  local error=""
+  local put_result=0
+  if [[ $1 == 's3api' ]]; then
+    log 5 "bucket name: $2, acls: $3"
+    error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$2" --access-control-policy "file://$3" 2>&1) || put_result=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$2" --acl-grant=read:ABCDEFG 2>&1) || put_result=$?
+  else
+    log 2 "put_bucket_acl not implemented for '$1'"
+    return 1
+  fi
+  if [[ $put_result -ne 0 ]]; then
+    log 2 "error putting bucket acl: $error"
+    return 1
+  fi
+  return 0
+}

tests/commands/put_bucket_policy.sh

@@ -2,21 +2,21 @@
 
 put_bucket_policy() {
   if [[ $# -ne 3 ]]; then
-    echo "get bucket policy command requires command type, bucket, policy file"
+    log 2 "get bucket policy command requires command type, bucket, policy file"
     return 1
   fi
-  if [[ $1 == 'aws' ]]; then
-    policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3") || get_result=$?
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
+    policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2") || get_result=$?
+    policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_result=$?
   elif [[ $1 == 'mc' ]]; then
-    policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2")
+    policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_result=$?
   else
-    echo "command 'put bucket policy' not implemented for '$1'"
+    log 2 "command 'put bucket policy' not implemented for '$1'"
     return 1
   fi
-  if [[ $get_result -ne 0 ]]; then
-    echo "error putting policy: $policy"
+  if [[ $put_result -ne 0 ]]; then
+    log 2 "error putting policy: $policy"
     return 1
   fi
   return 0

tests/commands/put_bucket_versioning.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+put_bucket_versioning() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "put bucket versioning command requires command type, bucket name, 'Enabled' or 'Suspended'"
+    return 1
+  fi
+  local put_result=0
+  if [[ $1 == 's3api' ]]; then
+    error=$(aws --no-verify-ssl s3api put-bucket-versioning --bucket "$2" --versioning-configuration "{ \"Status\": \"$3\"}" 2>&1) || put_result=$?
+  fi
+  if [[ $put_result -ne 0 ]]; then
+    log 2 "error putting bucket versioning: $error"
+    return 1
+  fi
+  return 0
+}

tests/logger.sh

@@ -10,7 +10,16 @@ log() {
   if [[ $1 -gt $LOG_LEVEL ]]; then
     return 0
   fi
-  echo "$2"
+  log_level=""
+  case "$1" in
+    1) log_level="CRIT";;
+    2) log_level="ERROR";;
+    3) log_level="WARN";;
+    4) log_level="INFO";;
+    5) log_level="DEBUG";;
+    6) log_level="TRACE";;
+  esac
+  echo "$log_level $2"
   if [[ -n "$TEST_LOG_FILE" ]]; then
     echo "$2" >> "$TEST_LOG_FILE"
   fi

tests/setup.sh

@@ -74,7 +74,7 @@ check_params() {
 # fail a test
 # param:  error message
 fail() {
-  echo "$1"
+  log 1 "$1"
   return 1
 }
 

tests/test_aws.sh

@@ -5,13 +5,18 @@ source ./tests/util.sh
 source ./tests/util_aws.sh
 source ./tests/util_bucket_create.sh
 source ./tests/util_file.sh
+source ./tests/util_users.sh
 source ./tests/test_common.sh
 source ./tests/commands/copy_object.sh
 source ./tests/commands/delete_bucket_policy.sh
 source ./tests/commands/delete_object_tagging.sh
+source ./tests/commands/get_bucket_acl.sh
 source ./tests/commands/get_bucket_policy.sh
+source ./tests/commands/get_bucket_versioning.sh
 source ./tests/commands/get_object.sh
+source ./tests/commands/put_bucket_acl.sh
 source ./tests/commands/put_bucket_policy.sh
+source ./tests/commands/put_bucket_versioning.sh
 source ./tests/commands/put_object.sh
 
 @test "test_abort_multipart_upload" {
@@ -55,6 +60,15 @@ source ./tests/commands/put_object.sh
   delete_test_files $bucket_file
 }
 
+@test "test_copy_object" {
+  test_common_copy_object "s3api"
+}
+
+# test creation and deletion of bucket on versitygw
+@test "test_create_delete_bucket_aws" {
+  test_common_create_delete_bucket "aws"
+}
+
 @test "test_put_object" {
   bucket_file="bucket_file"
 
@@ -78,11 +92,6 @@ source ./tests/commands/put_object.sh
   delete_test_files "$bucket_file"
 }
 
-# test creation and deletion of bucket on versitygw
-@test "test_create_delete_bucket_aws" {
-  test_common_create_delete_bucket "aws"
-}
-
 @test "test_create_bucket_invalid_name" {
   if [[ $RECREATE_BUCKETS != "true" ]]; then
     return
@@ -119,7 +128,7 @@ source ./tests/commands/put_object.sh
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local created=$?
   [[ $created -eq 0 ]] || fail "Error creating bucket"
 
-  get_bucket_acl "$BUCKET_ONE_NAME" || local result=$?
+  get_bucket_acl "s3api" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Error retrieving acl"
 
   id=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq '.Owner.ID')
@@ -128,6 +137,10 @@ source ./tests/commands/put_object.sh
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
 }
 
+@test "test_put_bucket_acl" {
+  test_common_put_bucket_acl "s3api"
+}
+
 # test ability to retrieve object ACLs
 #@test "test_get_object_acl" {
 
@@ -186,6 +199,10 @@ source ./tests/commands/put_object.sh
   test_common_set_get_delete_bucket_tags "aws"
 }
 
+#@test "test_get_set_versioning" {
+#  test_common_get_set_versioning "s3api"
+#}
+
 # test v1 s3api list objects command
 @test "test-s3api-list-objects-v1" {
   local object_one="test-file-one"

tests/test_common.sh

@@ -6,9 +6,12 @@ source ./tests/util_file.sh
 source ./tests/util_policy.sh
 source ./tests/commands/copy_object.sh
 source ./tests/commands/delete_object_tagging.sh
+source ./tests/commands/get_bucket_acl.sh
 source ./tests/commands/get_bucket_location.sh
 source ./tests/commands/get_bucket_tagging.sh
+source ./tests/commands/get_object.sh
 source ./tests/commands/list_buckets.sh
+source ./tests/commands/put_bucket_acl.sh
 source ./tests/commands/put_object.sh
 
 test_common_multipart_upload() {
@@ -53,6 +56,42 @@ test_common_create_delete_bucket() {
   [[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
 }
 
+test_common_copy_object() {
+  if [[ $# -ne 1 ]]; then
+    fail "copy object test requires command type"
+  fi
+  local object_name="test-object"
+  create_test_files "$object_name" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+  echo "test data" > "$test_file_folder/$object_name"
+
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket one"
+  setup_bucket "$1" "$BUCKET_TWO_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket two"
+
+  if [[ $1 == 's3' ]]; then
+    copy_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || local put_result=$?
+  else
+    put_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || local put_result=$?
+  fi
+  [[ $put_result -eq 0 ]] || fail "Failed to add object to bucket"
+  if [[ $1 == 's3' ]]; then
+    copy_object "$1" "s3://$BUCKET_ONE_NAME/$object_name" "$BUCKET_TWO_NAME" "$object_name" || local copy_result_one=$?
+  else
+    copy_object "$1" "$BUCKET_ONE_NAME/$object_name" "$BUCKET_TWO_NAME" "$object_name" || local copy_result_one=$?
+  fi
+  [[ $copy_result_one -eq 0 ]] || fail "Object not added to bucket"
+  get_object "$1" "$BUCKET_TWO_NAME" "$object_name" "$test_file_folder/$object_name-copy" || local get_result=$?
+  [[ $get_result -eq 0 ]] || fail "failed to retrieve object"
+
+  compare_files "$test_file_folder/$object_name" "$test_file_folder/$object_name-copy" || local compare_result=$?
+  [[ $compare_result -eq 0 ]] || fail "files not the same"
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_bucket_or_contents "$1" "$BUCKET_TWO_NAME"
+}
+
 test_common_put_object_with_data() {
   if [[ $# -ne 1 ]]; then
     fail "put object test requires command type"
@@ -98,6 +137,56 @@ test_common_put_object() {
   delete_test_files "$2"
 }
 
+test_common_put_get_object() {
+  if [[ $# -ne 1 ]]; then
+    fail "put, get object test requires command type"
+  fi
+
+  local object_name="test-object"
+  create_test_files "$object_name" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+  echo "test data" > "$test_file_folder"/"$object_name"
+
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+  put_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || local copy_result=$?
+  [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket"
+  object_exists "$1" "$BUCKET_ONE_NAME" "$object_name" || local exists_result_one=$?
+  [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
+
+  get_object "$1" "$BUCKET_ONE_NAME" "$object_name" "$test_file_folder/${object_name}_copy" || local delete_result=$?
+  [[ $delete_result -eq 0 ]] || fail "Failed to delete object"
+  object_exists "$1" "$BUCKET_ONE_NAME" "$object_name" || local exists_result_two=$?
+  [[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
+
+  compare_files "$test_file_folder"/"$object_name" "$test_file_folder/${object_name}_copy" || compare_result=$?
+  [[ $compare_result -ne 0 ]] || fail "objects are different"
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_test_files "$test_file_folder/$object_name" "$test_file_folder/${object_name}_copy"
+}
+
+test_common_get_set_versioning() {
+  local object_name="test-object"
+  create_test_files "$object_name" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+  get_bucket_versioning "$1" "$BUCKET_ONE_NAME" || local get_result=$?
+  [[ $get_result -eq 0 ]] || fail "error getting bucket versioning"
+
+  put_bucket_versioning "$1" "$BUCKET_ONE_NAME" "Enabled" || local put_result=$?
+  [[ $put_result -eq 0 ]] || fail "error putting bucket versioning"
+
+  get_bucket_versioning "$1" "$BUCKET_ONE_NAME" || local get_result=$?
+  [[ $get_result -eq 0 ]] || fail "error getting bucket versioning"
+
+  fail "test fail"
+}
+
 # common test for listing buckets
 # param:  "aws" or "s3cmd"
 # pass if buckets are properly listed, fail if not
@@ -361,6 +450,91 @@ test_common_get_bucket_location() {
   [[ $bucket_location == "null" ]] || [[ $bucket_location == "us-east-1" ]] || fail "wrong location: '$bucket_location'"
 }
 
+test_common_put_bucket_acl() {
+  [[ $# -eq 1 ]] || fail "test common put bucket acl missing command type"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating bucket"
+
+  if ! user_exists "ABCDEFG"; then
+    create_user "ABCDEFG" "HIJKLMN" user || create_result=$?
+    [[ $create_result -eq 0 ]] || fail "Error creating user"
+  fi
+
+  get_bucket_acl "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+
+  log 5 "Initial ACLs: $acl"
+  id=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq '.Owner.ID')
+  if [[ $id != '"'"$AWS_ACCESS_KEY_ID"'"' ]]; then
+    # in some cases, ID is canonical user ID rather than AWS_ACCESS_KEY_ID
+    canonical_id=$(aws --no-verify-ssl s3api list-buckets --query 'Owner.ID') || local list_result=$?
+    [[ $list_result -eq 0 ]] || fail "error getting canonical ID: $canonical_id"
+    [[ $id == "$canonical_id" ]] || fail "acl ID doesn't match AWS key or canonical ID"
+  fi
+
+  acl_file="test-acl"
+
+cat <<EOF > "$test_file_folder"/"$acl_file"
+  {
+    "Grants": [
+      {
+        "Grantee": {
+          "ID": "ABCDEFG",
+          "Type": "CanonicalUser"
+        },
+        "Permission": "READ"
+      }
+    ],
+    "Owner": {
+      "ID": "$AWS_ACCESS_KEY_ID"
+    }
+  }
+EOF
+
+  put_bucket_acl "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || local put_result=$?
+  [[ $put_result -eq 0 ]] || fail "Error putting acl"
+
+  get_bucket_acl "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+
+  log 5 "Acls after 1st put: $acl"
+  public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants[0]')
+  permission=$(echo "$public_grants" | jq -r '.Permission')
+  [[ $permission == "READ" ]] || fail "incorrect permission ($permission)"
+
+cat <<EOF > "$test_file_folder"/"$acl_file"
+  {
+    "Grants": [
+      {
+        "Grantee": {
+          "ID": "ABCDEFG",
+          "Type": "CanonicalUser"
+        },
+        "Permission": "FULL_CONTROL"
+      }
+    ],
+    "Owner": {
+      "ID": "$AWS_ACCESS_KEY_ID"
+    }
+  }
+EOF
+
+  put_bucket_acl "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || local put_result=$?
+  [[ $put_result -eq 0 ]] || fail "Error putting acl"
+
+  get_bucket_acl "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+
+  log 5 "Acls after 2nd put: $acl"
+  public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants')
+  public_grant_length=$(echo "$public_grants" | jq 'length')
+  [[ $public_grant_length -eq 1 ]] || fail "incorrect grant length for private ACL ($public_grant_length)"
+  permission=$(echo "$public_grants" | jq -r '.[0].Permission')
+  [[ $permission == "FULL_CONTROL" ]] || fail "incorrect permission ($permission)"
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+}
+
 test_common_get_put_delete_bucket_policy() {
   [[ $# -eq 1 ]] || fail "get/put/delete policy test requires command type"
 

tests/test_mc.sh

@@ -13,8 +13,12 @@ export RUN_MC=true
   test_common_multipart_upload "mc"
 }
 
+@test "test_copy_object" {
+  test_common_copy_object "mc"
+}
+
 # test mc bucket creation/deletion
-@test "test_create_delete_bucket_mc" {
+@test "test_create_delete_bucket" {
   test_common_create_delete_bucket "mc"
 }
 

tests/test_s3.sh

@@ -2,10 +2,18 @@
 
 source ./tests/test_common.sh
 
-@test "test_multipart_upload" {
+@test "test_complete_multipart_upload" {
   test_common_multipart_upload "s3"
 }
 
+@test "test_copy_object" {
+  test_common_copy_object "s3"
+}
+
+@test "test_create_delete_bucket" {
+  test_common_create_delete_bucket "s3"
+}
+
 @test "test_put_object" {
   test_common_put_object_no_data "s3"
 }
@@ -16,4 +24,8 @@ source ./tests/test_common.sh
 
 @test "test_list_objects_file_count" {
   test_common_list_objects_file_count "s3"
+}
+
+@test "test_put_get_object" {
+  test_common_put_get_object "s3"
 }

tests/test_s3cmd.sh

@@ -4,21 +4,17 @@ source ./tests/setup.sh
 source ./tests/test_common.sh
 source ./tests/util.sh
 source ./tests/util_bucket_create.sh
+source ./tests/util_users.sh
 source ./tests/commands/delete_bucket_policy.sh
 source ./tests/commands/get_bucket_policy.sh
 source ./tests/commands/put_bucket_policy.sh
 
 export RUN_S3CMD=true
 
-@test "test_multipart_upload_s3cmd" {
+@test "test_complete_multipart_upload" {
   test_common_multipart_upload "s3cmd"
 }
 
-# test s3cmd bucket creation/deletion
-@test "test_create_delete_bucket_s3cmd" {
-  test_common_create_delete_bucket "s3cmd"
-}
-
 # test s3cmd put object
 @test "test_copy_object_with_data" {
   test_common_put_object_with_data "s3cmd"
@@ -28,6 +24,15 @@ export RUN_S3CMD=true
   test_common_put_object_no_data "s3cmd"
 }
 
+# test s3cmd bucket creation/deletion
+@test "test_create_delete_bucket" {
+  test_common_create_delete_bucket "s3cmd"
+}
+
+#@test "test_put_bucket_acl" {
+#  test_common_put_bucket_acl "s3cmd"
+#}
+
 # test listing buckets on versitygw
 @test "test_list_buckets_s3cmd" {
   test_common_list_buckets "s3cmd"

tests/util.sh

@@ -17,7 +17,7 @@ source ./tests/commands/list_objects.sh
 # return 0 for success, 1 for failure
 delete_bucket_recursive() {
   if [ $# -ne 2 ]; then
-    echo "delete bucket missing command type, bucket name"
+    log 2 "delete bucket missing command type, bucket name"
     return 1
   fi
 
@@ -32,7 +32,7 @@ delete_bucket_recursive() {
   elif [[ $1 == "mc" ]]; then
     error=$(delete_bucket_recursive_mc "$2") || exit_code="$?"
   else
-    echo "invalid command type '$1'"
+    log 2 "invalid command type '$1'"
     return 1
   fi
 
@@ -40,7 +40,7 @@ delete_bucket_recursive() {
     if [[ "$error" == *"The specified bucket does not exist"* ]]; then
       return 0
     else
-      echo "error deleting bucket recursively: $error"
+      log 2 "error deleting bucket recursively: $error"
       return 1
     fi
   fi
@@ -79,24 +79,24 @@ delete_bucket_recursive_s3api() {
 # return 0 for success, 1 for failure
 delete_bucket_contents() {
   if [ $# -ne 2 ]; then
-    echo "delete bucket missing command id, bucket name"
+    log 2 "delete bucket missing command id, bucket name"
     return 1
   fi
 
   local exit_code=0
   local error
-  if [[ $1 == "aws" ]]; then
+  if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then
     error=$(aws --no-verify-ssl s3 rm s3://"$2" --recursive 2>&1) || exit_code="$?"
   elif [[ $1 == "s3cmd" ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate del s3://"$2" --recursive --force 2>&1) || exit_code="$?"
   elif [[ $1 == "mc" ]]; then
     error=$(mc --insecure rm --force --recursive "$MC_ALIAS"/"$2" 2>&1) || exit_code="$?"
   else
-    echo "invalid command type $1"
+    log 2 "invalid command type $1"
     return 1
   fi
   if [ $exit_code -ne 0 ]; then
-    echo "error deleting bucket contents: $error"
+    log 2 "error deleting bucket contents: $error"
     return 1
   fi
   return 0
@@ -414,23 +414,6 @@ object_is_accessible() {
   return 0
 }
 
-# get bucket acl
-# param:  bucket path
-# export acl for success, return 1 for error
-get_bucket_acl() {
-  if [ $# -ne 1 ]; then
-    echo "bucket ACL command missing bucket name"
-    return 1
-  fi
-  local exit_code=0
-  acl=$(aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1) || exit_code="$?"
-  if [ $exit_code -ne 0 ]; then
-    echo "Error getting bucket ACLs: $acl"
-    return 1
-  fi
-  export acl
-}
-
 # get object acl
 # param:  object path
 # export acl for success, return 1 for error
@@ -463,7 +446,7 @@ put_bucket_tag() {
   elif [[ $1 == 'mc' ]]; then
     error=$(mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$?
   else
-    echo "invalid command type $1"
+    log 2 "invalid command type $1"
     return 1
   fi
   if [[ $result -ne 0 ]]; then

tests/util_bucket_create.sh

@@ -5,7 +5,7 @@ source ./tests/logger.sh
 
 create_bucket_with_user() {
   if [ $# -ne 4 ]; then
-    echo "create bucket missing command type, bucket name, access, secret"
+    log 2 "create bucket missing command type, bucket name, access, secret"
     return 1
   fi
   local exit_code=0
@@ -16,11 +16,11 @@ create_bucket_with_user() {
   elif [[ $1 == "mc" ]]; then
     error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
-    echo "invalid command type $1"
+    log 2 "invalid command type $1"
     return 1
   fi
   if [ $exit_code -ne 0 ]; then
-    echo "error creating bucket: $error"
+    log 2 "error creating bucket: $error"
     export error
     return 1
   fi
@@ -29,7 +29,7 @@ create_bucket_with_user() {
 
 create_bucket_invalid_name() {
   if [ $# -ne 1 ]; then
-    echo "create bucket w/invalid name missing command type"
+    log 2 "create bucket w/invalid name missing command type"
     return 1
   fi
   local exit_code=0
@@ -42,11 +42,11 @@ create_bucket_invalid_name() {
   elif [[ $1 == 'mc' ]]; then
     bucket_create_error=$(mc --insecure mb "$MC_ALIAS" 2>&1) || exit_code=$?
   else
-    echo "invalid command type $1"
+    log 2 "invalid command type $1"
     return 1
   fi
   if [ $exit_code -eq 0 ]; then
-    echo "error:  bucket should have not been created but was"
+    log 2 "error:  bucket should have not been created but was"
     return 1
   fi
   export bucket_create_error

tests/util_file.sh

@@ -90,8 +90,14 @@ compare_files() {
     echo "file comparison requires two files"
     return 2
   fi
-  file_one_md5=$(md5 -q "$1")
-  file_two_md5=$(md5 -q "$2")
+  os=$(uname)
+  if [[ $os == "Darwin" ]]; then
+    file_one_md5=$(md5 -q "$1")
+    file_two_md5=$(md5 -q "$2")
+  else
+    file_one_md5=$(md5sum "$1" | cut -d " " -f 1)
+    file_two_md5=$(md5sum "$2" | cut -d " " -f 1)
+  fi
   if [[ $file_one_md5 == "$file_two_md5" ]]; then
     return 0
   fi