Merge pull request #1979 from versity/test/util_cleanup

test: cleanup of some common tests, bucket and file creation

tests/commands/create_bucket.sh

@@ -37,12 +37,12 @@ create_bucket() {
   elif [[ $1 == "mc" ]]; then
     error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" --region "$AWS_REGION" 2>&1) || exit_code=$?
   else
-    log 2 "invalid command type $1"
-    return 1
+    error="invalid command type $1"
+    exit_code=1
   fi
   if [ $exit_code -ne 0 ]; then
-    log 2 "error creating bucket: $error"
-    return 1
+    printf "error creating bucket: %s\n" "$error" >&2
+    return $exit_code
   fi
   return 0
 }

tests/commands/get_bucket_policy.sh

@@ -166,7 +166,7 @@ get_bucket_policy_mc() {
   if ! check_param_count "get_bucket_policy_mc" "bucket" 1 $#; then
     return 1
   fi
-  bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$?
+  bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1" 2>&1) || get_result=$?
   if [[ $get_result -ne 0 ]]; then
     log 2 "error getting policy: $bucket_policy"
     return 1

tests/commands/head_bucket.sh

@@ -14,6 +14,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+source ./tests/drivers/file.sh
 source ./tests/report.sh
 
 # params: client, bucket name
@@ -24,7 +25,7 @@ source ./tests/report.sh
 head_bucket() {
   log 6 "head_bucket '$1' '$2'"
   if ! check_param_count "head_bucket" "client, bucket name" 2 $#; then
-    return 1
+    return 2
   fi
   local exit_code=0
   if [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then
@@ -34,7 +35,7 @@ head_bucket() {
   elif [[ $1 == 'mc' ]]; then
     bucket_info=$(send_command mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'rest' ]]; then
-    bucket_info=$(head_bucket_rest "$2") || exit_code=$?
+    bucket_info=$(head_bucket_rest "$2" 2>&1) || exit_code=$?
     log 5 "head bucket rest exit code: $exit_code"
     return $exit_code
   else
@@ -42,12 +43,13 @@ head_bucket() {
   fi
   if [ $exit_code -ne 0 ]; then
     if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then
-      return 1
+      exit_code=1
+    else
+      exit_code=2
     fi
-    log 2 "error getting bucket info: $bucket_info"
-    return 2
+    echo "error getting bucket info: $bucket_info" >&2
+    return $exit_code
   fi
-  bucket_info="$(echo -n "$bucket_info" | grep -v "InsecureRequestWarning")"
   echo "$bucket_info"
   return 0
 }
@@ -57,20 +59,30 @@ head_bucket_rest() {
   if ! check_param_count_gt "bucket, callback, params (optional)" 1 $#; then
     return 2
   fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/head_bucket.sh 2>&1); then
+  if ! result_file=$(get_file_name 2>&1); then
+    log 2 "error getting result file: $result_file"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/$result_file" ./tests/rest_scripts/head_bucket.sh 2>&1); then
     log 2 "error getting head bucket: $result"
     return 2
   fi
+  local callback_code=0
+  if [ "$2" != "" ]; then
+    callback_result="$($2 "$result" "$TEST_FILE_FOLDER/$result_file" 2>&1)" || callback_code=$?
+    echo "$callback_result"
+    return "$callback_code"
+  fi
+  response_data="$(cat "$TEST_FILE_FOLDER/$result_file")"
   if [ "$result" == "200" ]; then
-    bucket_info="$(cat "$TEST_FILE_FOLDER/result.txt")"
-    echo "$bucket_info"
-    log 5 "bucket info: $bucket_info"
+    echo "$response_data"
     return 0
-  elif [ "$result" == "404" ]; then
-    log 5 "bucket '$1' not found"
+  fi
+  if [ "$result" == "404" ]; then
+    echo "$response_data"
     return 1
   fi
-  log 2 "unexpected response code '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+  echo "unexpected response code '$result' ($(cat "$TEST_FILE_FOLDER/$result_file"))" >&2
   return 2
 }
 

tests/drivers/create_bucket/create_bucket_rest.sh

@@ -158,6 +158,31 @@ setup_bucket_v2() {
   return 0
 }
 
+setup_bucket_v3() {
+  if ! check_param_count_v2 "bucket env var" 1 $#; then
+    return 1
+  fi
+  if ! error=$(bucket_cleanup_if_bucket_exists_v2 "$1" 2>&1); then
+    log 2 "error cleaning up bucket(s), if it/they exist(s): $error"
+    return 1
+  fi
+  if [ "$RECREATE_BUCKETS" == "false" ]; then
+    echo "$1"
+    return 0
+  else
+    if ! bucket_name=$(get_bucket_name "$1" 2>&1); then
+      log 2 "error getting bucket name: $bucket_name"
+      return 1
+    fi
+    if ! error=$(create_bucket_rest_expect_success "$bucket_name" "" 2>&1); then
+      log 2 "error creating bucket named '$bucket_name': $error"
+      return 1
+    fi
+  fi
+  echo "$bucket_name"
+  return 0
+}
+
 # params:  client, bucket name(s)
 # return 0 for success, 1 for failure
 setup_buckets() {
@@ -186,6 +211,22 @@ setup_buckets_v2() {
   return 0
 }
 
+setup_buckets_v3() {
+  if ! check_param_count_gt "minimum of 1 bucket env var" 1 $#; then
+    return 1
+  fi
+  local buckets=()
+  for name in "$@"; do
+    if ! bucket=$(setup_bucket_v3 "$name" 2>&1); then
+      log 2 "error setting up bucket '$name': $bucket"
+      return 1
+    fi
+    buckets+=("$bucket")
+  done
+  echo "${buckets[*]}"
+  return 0
+}
+
 get_bucket_name() {
   if ! check_param_count_v2 "bucket" 1 $#; then
     return 1
@@ -194,7 +235,14 @@ get_bucket_name() {
     echo "$1"
     return 0
   fi
-  echo "$1-$(date +%Y%m%d%H%M%S)"
+  if ! uuid=$(uuidgen 2>&1); then
+    log 2 "error getting UUID: $uuid"
+    return 1
+  fi
+  local bucket_name
+  bucket_name="$1-${uuid,,}"
+  echo "${bucket_name:0:63}"
+  return 0
 }
 
 setup_bucket_object_lock_enabled_v2() {

tests/drivers/file.sh

@@ -33,12 +33,28 @@ setup_bucket_and_file_v2() {
     return 1
   fi
   if ! setup_bucket_and_file_base "$1" "setup_bucket_and_files_v2" "$2"; then
-    log 2 "error setting up bucket and files"
+    log 2 "error setting up bucket and file"
     return 1
   fi
   return 0
 }
 
+setup_bucket_and_file_v3() {
+  if ! check_param_count_v2 "bucket env var" 1 $#; then
+    return 1
+  fi
+  if ! bucket_name=$(setup_bucket_v3 "$1" 2>&1); then
+    log 2 "error setting up bucket and file: $bucket_name"
+    return 1
+  fi
+  if ! test_file=$(create_test_file_v2 2>&1); then
+    log 2 "error creating test file: $test_file"
+    return 1
+  fi
+  echo "$bucket_name $test_file"
+  return 0
+}
+
 setup_bucket_and_file_base() {
   if ! check_param_count_v2 "bucket, function, file name" 3 $#; then
     return 1
@@ -72,6 +88,27 @@ setup_bucket_and_files_v2() {
   return 0
 }
 
+setup_bucket_and_files_v3() {
+  if ! check_param_count_v2 "bucket env var, file count" 2 $#; then
+    return 1
+  fi
+  local bucket_and_files=()
+  if ! bucket_name=$(setup_bucket_v3 "$1" 2>&1); then
+    log 2 "error setting up bucket"
+    return 1
+  fi
+  bucket_and_files=("$bucket_name")
+  for ((i=0;i<$2;i++)); do
+    if ! file_name=$(create_test_file_v2 2>&1); then
+      log 2 "error creating test file: $file_name"
+      return 1
+    fi
+    bucket_and_files+=("$file_name")
+  done
+  echo "${bucket_and_files[*]}"
+  return 0
+}
+
 setup_bucket_and_files_base() {
   if ! check_param_count_gt "bucket, setup bucket function, file name" 3 $#; then
     return 1
@@ -124,6 +161,22 @@ setup_bucket_and_large_file_v2() {
   return 0
 }
 
+setup_bucket_and_large_file_v3() {
+  if ! check_param_count_v2 "bucket env var" 1 $#; then
+    return 1
+  fi
+  if ! bucket_name=$(setup_bucket_v3 "$1" 2>&1); then
+    log 2 "error setting up bucket: $bucket_name"
+    return 1
+  fi
+  if ! file_name=$(create_large_file_v2 "$file_name" 2>&1); then
+    log 2 "error creating large file: $file_name"
+    return 1
+  fi
+  echo "$bucket_name $file_name"
+  return 0
+}
+
 chunked_upload_trailer_success() {
   if ! check_param_count_v2 "checksum" 1 $#; then
     return 1
@@ -204,3 +257,34 @@ create_folder_if_needed_and_file() {
   fi
   return 0
 }
+
+# optional parameter - file size
+# shellcheck disable=SC2120
+create_test_file_v2() {
+  if ! file_name=$(get_file_name 2>&1); then
+    log 2 "error getting file name: $file_name"
+    return 1
+  fi
+  if ! error=$(create_test_file "$file_name" "$1" 2>&1); then
+    log 2 "error creating test file: $error"
+    return 1
+  fi
+  echo "$file_name"
+  return 0
+}
+
+get_file_names() {
+  if ! check_param_count_v2 "file name count" 1 $#; then
+    return 1
+  fi
+  file_names=()
+  for ((i=0;i<$1;i++)); do
+    if ! file_name=$(get_file_name 2>&1); then
+      log 2 "error getting file name: $file_name"
+      return 1
+    fi
+    file_names+=("$file_name")
+  done
+  echo "${file_names[*]}"
+  return 0
+}

tests/drivers/head_bucket/head_bucket_rest.sh

@@ -18,16 +18,23 @@ bucket_exists() {
   if ! check_param_count "bucket_exists" "bucket name" 1 $#; then
     return 2
   fi
-  local exists=0
-  head_bucket "rest" "$1" || exists=$?
-  log 5 "bucket exists response code: $exists"
-  # shellcheck disable=SC2181
-  if [ $exists -eq 2 ]; then
-    log 2 "unexpected error checking if bucket exists"
-    return 2
+  local response_code=0
+  exists=$(head_bucket_rest "$1" "check_bucket_existence_callback" 2>&1) || response_code=$?
+  echo "$exists"
+  return "$response_code"
+}
+
+check_bucket_existence_callback() {
+  if ! check_param_count_v2 "response code, response data" 2 $#; then
+    return 1
   fi
-  if [ $exists -eq 0 ]; then
+  if [ "$1" -eq 200 ]; then
+    echo "true"
     return 0
+  elif [ "$1" -eq 404 ]; then
+    echo "false"
+    return 1
   fi
-  return 1
+  echo "error checking if bucket exists (data: $2)"
+  return 2
 }

tests/setup_mc.sh

@@ -14,21 +14,64 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# return 0 for match, 1 for set or replace, 2 for error
 check_for_alias() {
-  local alias_result
-  aliases=$(mc alias list)
-  if [[ $alias_result -ne 0 ]]; then
+  if ! aliases=$(mc alias list 2>&1); then
     log 2 "error checking for aliases: $aliases"
     return 2
   fi
+  local alias_match="false" check_result=0
   while IFS= read -r line; do
-    if echo "$line" | grep -w "$MC_ALIAS "; then
+    check_mc_aliases_line "$line" "$alias_match" || local check_result=$?
+    if [ "$check_result" -eq 4 ]; then
+      return 2
+    elif [ "$check_result" -eq 3 ]; then
+      continue
+    elif [ "$check_result" -eq 2 ]; then
+      return 1
+    elif [ "$check_result" -eq 1 ]; then
       return 0
+    else
+      alias_match="true"
     fi
   done <<< "$aliases"
   return 1
 }
 
+# return 0 for alias name match, 1 for alias key match, 2 for alias key mismatch, 3 for keep looking, 4 for error
+check_mc_aliases_line() {
+  if ! check_param_count_v2 "line, alias found" 2 $#; then
+    return 4
+  fi
+  if [ "$2" == "true" ]; then
+    check_alias_access_key "$1" || local check_result=$?
+    return "$((check_result+1))"
+  else
+    if echo "$1" | grep -w "$MC_ALIAS "; then
+      return 0
+    fi
+  fi
+}
+
+# return 0 for match, 1 for mismatch, 2 for keep looking, 3 for error
+check_alias_access_key() {
+  if ! check_param_count_v2 "line" 1 $#; then
+    return 3
+  fi
+  if [[ "$1" =~ ^[[:space:]]*$ ]]; then
+    return 3
+  fi
+  alias_access_key_id=$(echo -n "$1" | awk -F': *' '/^AccessKey[[:space:]]*:/ {print $2; exit}')
+  if [ "$alias_access_key_id" == "" ]; then
+    return 2
+  fi
+  if [ "$alias_access_key_id" == "$AWS_ACCESS_KEY_ID" ]; then
+    return 0
+  fi
+  return 1
+}
+
+
 check_add_mc_alias() {
   check_for_alias || alias_result=$?
   if [[ $alias_result -eq 2 ]]; then
@@ -45,4 +88,4 @@ check_add_mc_alias() {
     return 1
   fi
   return 0
-}
+}

tests/test_common.sh

@@ -29,6 +29,7 @@ source ./tests/commands/put_bucket_tagging.sh
 source ./tests/commands/put_object_tagging.sh
 source ./tests/commands/put_object.sh
 source ./tests/commands/put_public_access_block.sh
+source ./tests/drivers/create_bucket/create_bucket_rest.sh
 source ./tests/drivers/file.sh
 source ./tests/drivers/params.sh
 source ./tests/util/util_file.sh
@@ -43,16 +44,16 @@ test_common_multipart_upload() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  bucket_file="largefile"
-  run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$bucket_file"
+  run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME"
   assert_success
+  read -r bucket_name bucket_file <<< "$output"
 
   if [ "$1" == 's3' ]; then
     run copy_file_locally "$TEST_FILE_FOLDER/$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
     assert_success
   fi
 
-  run put_object "$1" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
+  run put_object "$1" "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file"
   assert_success
 
   if [ "$1" == 's3' ]; then
@@ -60,7 +61,8 @@ test_common_multipart_upload() {
     assert_success
   fi
 
-  run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
+  log 5 "file: $TEST_FILE_FOLDER/$bucket_file, bucket: $bucket_name"
+  run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
   assert_success
 }
 
@@ -71,49 +73,58 @@ test_common_create_delete_bucket() {
   if [[ $RECREATE_BUCKETS != "true" ]]; then
     return
   fi
-
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"
+  run bucket_cleanup_if_bucket_exists_v2 "$BUCKET_ONE_NAME"
   assert_success
 
-  run create_bucket "$1" "$BUCKET_ONE_NAME"
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run create_bucket "$1" "$bucket_name"
   assert_success
 
-  run bucket_exists "$BUCKET_ONE_NAME"
+  run bucket_exists "$bucket_name"
+  assert_success
+  assert_output "true"
+
+  run delete_bucket "$1" "$bucket_name"
   assert_success
 
-  run delete_bucket "$1" "$BUCKET_ONE_NAME"
-  assert_success
+  run bucket_exists "$bucket_name"
+  assert_failure 1
+  assert_output "false"
 }
 
 test_common_copy_object() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  local object_name="test-object"
-  run create_test_file "$object_name"
+  run create_test_file_v2
   assert_success
+  object_name=$output
 
-  run setup_buckets "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
+  run setup_buckets_v3 "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
   assert_success
+  read -r bucket_one bucket_two <<< "$output"
 
   if [[ $1 == 's3' ]]; then
-    run copy_object "$1" "$TEST_FILE_FOLDER/$object_name" "$BUCKET_ONE_NAME" "$object_name"
+    run copy_object "$1" "$TEST_FILE_FOLDER/$object_name" "$bucket_one" "$object_name"
     assert_success
   else
-    run put_object "$1" "$TEST_FILE_FOLDER/$object_name" "$BUCKET_ONE_NAME" "$object_name"
+    run put_object "$1" "$TEST_FILE_FOLDER/$object_name" "$bucket_one" "$object_name"
     assert_success
   fi
   if [[ $1 == 's3' ]]; then
-    run copy_object "$1" "s3://$BUCKET_ONE_NAME/$object_name" "$BUCKET_TWO_NAME" "$object_name"
+    run copy_object "$1" "s3://$bucket_one/$object_name" "$bucket_two" "$object_name"
     assert_success
   else
-    run copy_object "$1" "$BUCKET_ONE_NAME/$object_name" "$BUCKET_TWO_NAME" "$object_name"
+    run copy_object "$1" "$bucket_one/$object_name" "$bucket_two" "$object_name"
     assert_success
   fi
-  run download_and_compare_file "$TEST_FILE_FOLDER/$object_name" "$BUCKET_TWO_NAME" "$object_name" "$TEST_FILE_FOLDER/$object_name-copy"
+  run download_and_compare_file "$TEST_FILE_FOLDER/$object_name" "$bucket_two" "$object_name" "$TEST_FILE_FOLDER/$object_name-copy"
   assert_success
 }
 
@@ -123,9 +134,9 @@ test_common_put_object_with_data() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  local object_name="test-object"
-  run create_test_file "$object_name"
+  run create_test_file_v2
   assert_success
+  object_name=$output
 
   test_common_put_object "$1" "$object_name"
 }
@@ -136,9 +147,9 @@ test_common_put_object_no_data() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  local object_name="test-object"
-  run create_test_file "$object_name" 0
+  run create_test_file_v2 0
   assert_success
+  object_name="$output"
 
   test_common_put_object "$1" "$object_name"
 }
@@ -149,8 +160,9 @@ test_common_put_object() {
   run assert_param_count "client type, file" 2 "$#"
   assert_success
 
-  run setup_bucket "$BUCKET_ONE_NAME"
+  run setup_bucket_v3 "$BUCKET_ONE_NAME"
   assert_success
+  bucket_name=$output
 
   # s3 erases file locally, so we need to copy it first
   if [ "$1" == 's3' ]; then
@@ -158,7 +170,7 @@ test_common_put_object() {
     assert_success
   fi
 
-  run put_object "$1" "$TEST_FILE_FOLDER/$2" "$BUCKET_ONE_NAME" "$2"
+  run put_object "$1" "$TEST_FILE_FOLDER/$2" "$bucket_name" "$2"
   assert_success
 
   if [ "$1" == 's3' ]; then
@@ -166,13 +178,13 @@ test_common_put_object() {
     assert_success
   fi
 
-  run download_and_compare_file "$TEST_FILE_FOLDER/$2" "$BUCKET_ONE_NAME" "$2" "$TEST_FILE_FOLDER/${2}-copy"
+  run download_and_compare_file "$TEST_FILE_FOLDER/$2" "$bucket_name" "$2" "$TEST_FILE_FOLDER/${2}-copy"
   assert_success
 
-  run delete_object "$1" "$BUCKET_ONE_NAME" "$2"
+  run delete_object "$1" "$bucket_name" "$2"
   assert_success
 
-  run object_exists "$1" "$BUCKET_ONE_NAME" "$2"
+  run object_exists "$1" "$bucket_name" "$2"
   assert_failure 1
 }
 
@@ -208,10 +220,11 @@ test_common_list_buckets() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  run setup_buckets "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
+  run setup_buckets_v3 "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
   assert_success
+  read -r bucket_one bucket_two <<< "$output"
 
-  run list_and_check_buckets "$1" "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
+  run list_and_check_buckets "$1" "$bucket_one" "$bucket_two"
   assert_success
 }
 
@@ -219,18 +232,17 @@ test_common_list_objects() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  object_one="test-file-one"
-  object_two="test-file-two"
-  run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two"
+  run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2
+  assert_success
+  read -r bucket_name object_one object_two <<< "$output"
+
+  run put_object "$1" "$TEST_FILE_FOLDER/$object_one" "$bucket_name" "$object_one"
   assert_success
 
-  run put_object "$1" "$TEST_FILE_FOLDER"/$object_one "$BUCKET_ONE_NAME" "$object_one"
+  run put_object "$1" "$TEST_FILE_FOLDER/$object_two" "$bucket_name" "$object_two"
   assert_success
 
-  run put_object "$1" "$TEST_FILE_FOLDER"/$object_two "$BUCKET_ONE_NAME" "$object_two"
-  assert_success
-
-  run list_check_objects_common "$1" "$BUCKET_ONE_NAME" "$object_one" "$object_two"
+  run list_check_objects_common "$1" "$bucket_name" "$object_one" "$object_two"
   assert_success
 }
 
@@ -241,22 +253,23 @@ test_common_set_get_delete_bucket_tags() {
   local key="test_key"
   local value="test_value"
 
-  run setup_bucket "$BUCKET_ONE_NAME"
+  run setup_bucket_v3 "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name=$output
+
+  run verify_no_bucket_tags "$1" "$bucket_name"
   assert_success
 
-  run verify_no_bucket_tags "$1" "$BUCKET_ONE_NAME"
+  run put_bucket_tagging "$1" "$bucket_name" $key $value
   assert_success
 
-  run put_bucket_tagging "$1" "$BUCKET_ONE_NAME" $key $value
+  run get_and_check_bucket_tags "$bucket_name" "$key" "$value"
   assert_success
 
-  run get_and_check_bucket_tags "$BUCKET_ONE_NAME" "$key" "$value"
+  run delete_bucket_tagging "$1" "$bucket_name"
   assert_success
 
-  run delete_bucket_tagging "$1" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run verify_no_bucket_tags "$1" "$BUCKET_ONE_NAME"
+  run verify_no_bucket_tags "$1" "$bucket_name"
   assert_success
 }
 
@@ -264,23 +277,23 @@ test_common_set_get_object_tags() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  local bucket_file="bucket-file"
+  run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
+  assert_success
+  read -r bucket_name bucket_file <<< "$output"
+
   local key="test_key"
   local value="test_value"
 
-  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file"
+  run put_object "$1" "$TEST_FILE_FOLDER"/"$bucket_file" "$bucket_name" "$bucket_file"
   assert_success
 
-  run put_object "$1" "$TEST_FILE_FOLDER"/"$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
+  run verify_no_object_tags "$1" "$bucket_name" "$bucket_file"
   assert_success
 
-  run verify_no_object_tags "$1" "$BUCKET_ONE_NAME" "$bucket_file"
+  run put_object_tagging "$1" "$bucket_name" "$bucket_file" "$key" "$value"
   assert_success
 
-  run put_object_tagging "$1" "$BUCKET_ONE_NAME" $bucket_file $key $value
-  assert_success
-
-  run check_verify_object_tags "$1" "$BUCKET_ONE_NAME" "$bucket_file" "$key" "$value"
+  run check_verify_object_tags "$1" "$bucket_name" "$bucket_file" "$key" "$value"
   assert_success
 }
 
@@ -288,21 +301,25 @@ test_common_presigned_url_utf8_chars() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  local bucket_file="my-$%^&*;"
-  local bucket_file_copy="bucket-file-copy"
+  run get_file_names 2
+  assert_success
+  read -r file_header bucket_file_copy <<< "$output"
 
+  local bucket_file="${file_header}-$%^&*;"
   run create_test_file "$bucket_file"
   assert_success
+
   run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
   assert_success
 
-  run setup_bucket "$BUCKET_ONE_NAME"
+  run setup_bucket_v3 "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run put_object "$1" "$TEST_FILE_FOLDER"/"$bucket_file" "$bucket_name" "$bucket_file"
   assert_success
 
-  run put_object "$1" "$TEST_FILE_FOLDER"/"$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
-  assert_success
-
-  run create_check_presigned_url "$1" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file_copy"
+  run create_check_presigned_url "$1" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file_copy"
   assert_success
 
   run compare_files "$TEST_FILE_FOLDER"/"$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file_copy"
@@ -316,13 +333,14 @@ test_common_list_objects_file_count() {
   run create_test_file_count 1001
   assert_success
 
-  run setup_bucket "$BUCKET_ONE_NAME"
+  run setup_bucket_v3 "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name=$output
+
+  run put_object_multiple "$1" "$TEST_FILE_FOLDER/file_*" "$bucket_name"
   assert_success
 
-  run put_object_multiple "$1" "$TEST_FILE_FOLDER/file_*" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run list_objects_check_file_count "$1" "$BUCKET_ONE_NAME" 1001
+  run list_objects_check_file_count "$1" "$bucket_name" 1001
   assert_success
 }
 
@@ -330,26 +348,26 @@ test_common_delete_object_tagging() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  bucket_file="bucket_file"
   tag_key="key"
   tag_value="value"
 
-  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file"
+  run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
+  assert_success
+  read -r bucket_name bucket_file <<< "$output"
+
+  run put_object "$1" "$TEST_FILE_FOLDER"/"$bucket_file" "$bucket_name" "$bucket_file"
   assert_success
 
-  run put_object "$1" "$TEST_FILE_FOLDER"/"$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
+  run put_object_tagging "$1" "$bucket_name" "$bucket_file" "$tag_key" "$tag_value"
   assert_success
 
-  run put_object_tagging "$1" "$BUCKET_ONE_NAME" "$bucket_file" "$tag_key" "$tag_value"
+  run get_and_verify_object_tags "$1" "$bucket_name" "$bucket_file" "$tag_key" "$tag_value"
   assert_success
 
-  run get_and_verify_object_tags "$1" "$BUCKET_ONE_NAME" "$bucket_file" "$tag_key" "$tag_value"
+  run delete_object_tagging "$1" "$bucket_name" "$bucket_file"
   assert_success
 
-  run delete_object_tagging "$1" "$BUCKET_ONE_NAME" "$bucket_file"
-  assert_success
-
-  run check_object_tags_empty "$1" "$BUCKET_ONE_NAME" "$bucket_file"
+  run check_object_tags_empty "$1" "$bucket_name" "$bucket_file"
   assert_success
 }
 
@@ -357,10 +375,11 @@ test_common_get_bucket_location() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  run setup_bucket "$BUCKET_ONE_NAME"
+  run setup_bucket_v3 "$BUCKET_ONE_NAME"
   assert_success
+  bucket_name=$output
 
-  run get_check_bucket_location_various "$1" "$BUCKET_ONE_NAME"
+  run get_check_bucket_location_various "$1" "$bucket_name"
   assert_success
 }
 
@@ -368,36 +387,32 @@ test_common_get_put_delete_bucket_policy() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  policy_file="policy_file"
-
-  run create_test_file "$policy_file"
+  run setup_bucket_v3 "$BUCKET_ONE_NAME"
   assert_success
+  bucket_name=$output
 
   effect="Allow"
   principal="*"
   action="s3:GetObject"
-  resource="arn:aws:s3:::$BUCKET_ONE_NAME/*"
+  resource="arn:aws:s3:::$bucket_name/*"
 
-  run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
+  run setup_policy_with_single_statement_v2 "2012-10-17" "$effect" "$principal" "$action" "$resource"
   assert_success
-  log 5 "POLICY: $(cat "$TEST_FILE_FOLDER/$policy_file")"
+  policy_file="$output"
 
-  run setup_bucket "$BUCKET_ONE_NAME"
+  run check_for_empty_policy "$1" "$bucket_name"
   assert_success
 
-  run check_for_empty_policy "$1" "$BUCKET_ONE_NAME"
+  run put_bucket_policy "$1" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"
   assert_success
 
-  run put_bucket_policy "$1" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$policy_file"
+  run get_and_check_policy "$1" "$bucket_name" "$effect" "$principal" "$action" "$resource"
   assert_success
 
-  run get_and_check_policy "$1" "$BUCKET_ONE_NAME" "$effect" "$principal" "$action" "$resource"
+  run delete_bucket_policy "$1" "$bucket_name"
   assert_success
 
-  run delete_bucket_policy "$1" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run check_for_empty_policy "$1" "$BUCKET_ONE_NAME"
+  run check_for_empty_policy "$1" "$bucket_name"
   assert_success
 }
 
@@ -405,22 +420,22 @@ test_common_ls_directory_object() {
   run assert_param_count "client type" 1 "$#"
   assert_success
 
-  test_file="a"
-
-  run create_test_file "$test_file" 0
+  run create_test_file_v2 0
   assert_success
+  test_file="$output"
 
-  run setup_bucket "$BUCKET_ONE_NAME"
+  run setup_bucket_v3 "$BUCKET_ONE_NAME"
   assert_success
+  bucket_name="$output"
 
   if [ "$1" == 's3cmd' ]; then
     put_object_client="s3api"
   else
     put_object_client="$1"
   fi
-  run put_object "$put_object_client" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file/"
+  run put_object "$put_object_client" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file/"
   assert_success "error putting test file folder"
 
-  run list_and_check_directory_obj "$1" "$test_file"
+  run list_and_check_directory_obj "$1" "$bucket_name" "$test_file"
   assert_success "error listing and checking directory object"
 }

tests/test_s3.sh

@@ -17,7 +17,9 @@
 load ./bats-support/load
 load ./bats-assert/load
 
+source ./tests/drivers/create_bucket/create_bucket_rest.sh
 source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
+source ./tests/drivers/file.sh
 source ./tests/util/util_file.sh
 source ./tests/test_common.sh
 

tests/util/util_file.sh

@@ -219,6 +219,19 @@ create_large_file() {
   return 0
 }
 
+create_large_file_v2() {
+  if ! file_name=$(get_file_name 2>&1); then
+    log 2 "error getting file name: $file_name"
+    return 1
+  fi
+  if ! create_large_file_with_size "$file_name" 160; then
+    log 2 "error creating 160MB file with name '$file_name'"
+    return 1
+  fi
+  echo "$file_name"
+  return 0
+}
+
 create_large_file_with_size() {
   if ! check_param_count_v2 "file name, size in MB" 2 $#; then
     return 1

tests/util/util_object.sh

@@ -192,10 +192,10 @@ copy_file() {
 
 list_and_check_directory_obj() {
   log 6 "list_and_check_directory_obj"
-  if ! check_param_count "list_and_check_directory_obj" "client, file name" 2 $#; then
+  if ! check_param_count "list_and_check_directory_obj" "client, bucket, file name" 3 $#; then
     return 1
   fi
-  if ! list_objects_with_prefix "$1" "$BUCKET_ONE_NAME" "$2/"; then
+  if ! list_objects_with_prefix "$1" "$2" "$3/"; then
     log 2 "error listing objects with prefix"
     return 1
   fi
@@ -205,15 +205,15 @@ list_and_check_directory_obj() {
       log 2 "error getting key: $key"
       return 1
     fi
-    if [ "$key" != "$2/" ]; then
-      log 2 "key mismatch ($key, $2)"
+    if [ "$key" != "$3/" ]; then
+      log 2 "key mismatch ($key, $3)"
       return 1
     fi
   elif [ "$1" == "s3" ]; then
     log 5 "$objects"
     filename=$(echo "$objects" | grep -v "InsecureRequestWarning" | awk '{print $4}')
-    if [ "$filename" != "$2" ]; then
-      log 2 "filename mismatch ($filename, $2)"
+    if [ "$filename" != "$3" ]; then
+      log 2 "filename mismatch ($filename, $3)"
       return 1
     fi
   fi

tests/util/util_policy.sh

@@ -130,6 +130,22 @@ setup_policy_with_single_statement() {
   log 5 "policy data: $(cat "$1")"
 }
 
+setup_policy_with_single_statement_v2() {
+  if ! check_param_count_v2 "version, effect, principal, action, resource" 5 $#; then
+    return 1
+  fi
+  if ! policy_file=$(get_file_name 2>&1); then
+    log 2 "error getting policy file name"
+    return 1
+  fi
+  if ! setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "$1" "$2" "$3" "$4" "$5"; then
+    log 2 "error setting up policy"
+    return 1
+  fi
+  echo "$policy_file"
+  return 0
+}
+
 # params:  file, version, two sets:  effect, principal, action, resource
 # return 0 on success, 1 on error
 setup_policy_with_double_statement() {
@@ -174,7 +190,7 @@ get_and_check_policy() {
   if ! check_param_count "get_and_check_policy" "client, bucket, expected effect, principal, action, resource" 6 $#; then
     return 1
   fi
-  if ! get_bucket_policy "$1" "$BUCKET_ONE_NAME"; then
+  if ! get_bucket_policy "$1" "$2"; then
     log 2 "error getting bucket policy after setting"
     return 1
   fi