From dff20b5b9da0dd8cdc366c09b5616cdbc0fec1ae Mon Sep 17 00:00:00 2001
From: Luke McCrone <lrm110484@gmail.com>
Date: Mon, 29 Dec 2025 12:46:20 -0300
Subject: [PATCH] test: test matrix setup overhaul, fix some region/iam issues

---
 .github/workflows/system.yml                  | 214 +----------
 tests/commands/create_bucket.sh               |   2 +-
 tests/commands/get_bucket_location.sh         |  14 +-
 .../get_bucket_location.sh                    |   9 +-
 tests/env.sh                                  |   2 +-
 tests/generate_matrix.sh                      |  92 +++++
 tests/run.sh                                  | 351 ++++--------------
 tests/run_all.sh                              |   8 +-
 tests/test_rest_delete_bucket.sh              |  18 +
 tests/test_rest_put_bucket_policy.sh          |   3 +
 tests/test_s3api_bucket.sh                    |   3 +
 tests/test_s3cmd.sh                           |   3 +
 tests/util/util_users.sh                      |   9 +-
 tests/versity.sh                              |   2 +-
 14 files changed, 235 insertions(+), 495 deletions(-)
 create mode 100755 tests/generate_matrix.sh

diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml
index fcbea4e..28ef435 100644
--- a/.github/workflows/system.yml
+++ b/.github/workflows/system.yml
@@ -2,206 +2,28 @@ name: system tests
 permissions: {}
 on: pull_request
 jobs:
+  generate:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.make.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: make
+        run: |
+          if ! matrix_output=$(tests/generate_matrix.sh 2>&1); then
+            echo "error generating matrix: $matrix_output"
+            exit 1
+          fi
+          MATRIX_JSON=$(echo -n "$matrix_output" | jq -c . )
+          echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT"
+
   build:
     name: RunTests
+    needs: generate
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:
-        include:
-          - set: "mc, posix, non-file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "mc-non-file-count"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "mc, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "mc-file-count"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "REST, posix, non-static, base|acl|multipart|put-object, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-base,rest-acl,rest-multipart,rest-put-object"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "REST, posix, non-static, chunked|checksum|versioning|bucket, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket,rest-list-buckets,rest-create-bucket,rest-head-bucket"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "REST, posix, non-static, not implemented|rest-delete-bucket-ownership-controls|rest-delete-bucket-tagging, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-not-implemented,rest-delete-bucket-ownership-controls,rest-delete-bucket-tagging"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "REST, posix, static, base|acl|multipart|put-object, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-base,rest-acl,rest-multipart,rest-put-object"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "REST, posix, static, chunked|checksum|versioning|bucket, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket,rest-list-buckets,rest-create-bucket,rest-head-bucket"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "REST, posix, static, not implemented|rest-delete-bucket-ownership-controls|rest-delete-bucket-tagging, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-not-implemented,rest-delete-bucket-ownership-controls,rest-delete-bucket-tagging"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "REST, posix, static, rest-put-bucket-tagging|rest-get-bucket-location|rest-put-object-tagging, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-put-bucket-tagging,rest-get-bucket-location,rest-put-object-tagging,rest-get-object-tagging,rest-list-object-versions"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "REST, posix, non-static, rest-put-bucket-tagging|rest-get-bucket-location|rest-put-object-tagging, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-put-bucket-tagging,rest-get-bucket-location,rest-put-object-tagging,rest-get-object-tagging,rest-list-object-versions"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-west-1"
-          - set: "REST, posix, static, rest-get-bucket-policy-status, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-get-bucket-policy-status"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "REST, posix, non-static, rest-get-bucket-policy-status, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "rest-put-bucket-tagging,rest-get-bucket-location,rest-put-object-tagging,rest-get-object-tagging,rest-list-object-versions"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-west-1"
-          - set: "s3, posix, non-file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-non-file-count"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-file-count"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3api, posix, policy, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3api, posix, user, non-static, s3 IAM"
-            IAM_TYPE: s3
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3api, posix, bucket, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3api, posix, multipart, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-multipart"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3api, posix, object, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-object"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3api, posix, policy, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3api, posix, user, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "false"
-            DELETE_BUCKETS_AFTER_TEST: "false"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          # TODO fix/debug s3 gateway
-          #- set: "s3api, s3, multipart|object, non-static, folder IAM"
-          #  IAM_TYPE: folder
-          #  RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-          #  RECREATE_BUCKETS: "true"
-          #  BACKEND: "s3"
-          #- set: "s3api, s3, policy|user, non-static, folder IAM"
-          #  IAM_TYPE: folder
-          #  RUN_SET: "s3api-policy,s3api-user"
-          #  RECREATE_BUCKETS: "true"
-          #  BACKEND: "s3"
-          - set: "s3cmd, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-file-count"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3cmd, posix, non-user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-non-user"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "s3cmd, posix, user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-user"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
-          - set: "setup/remove static buckets scripts"
-            IAM_TYPE: folder
-            RUN_SET: "setup-remove-static"
-            RECREATE_BUCKETS: "true"
-            DELETE_BUCKETS_AFTER_TEST: "true"
-            BACKEND: "posix"
-            AWS_REGION: "us-east-1"
+      matrix: ${{ fromJson(needs.generate.outputs.matrix) }}
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v5
@@ -301,7 +123,7 @@ jobs:
           if [[ $RECREATE_BUCKETS == "false" ]]; then
             BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
           fi
-          BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
+          BYPASS_ENV_FILE=true $HOME/bin/bats ${{ github.workspace }}/$RUN_SET
 
       - name: Time report
         run: | 
diff --git a/tests/commands/create_bucket.sh b/tests/commands/create_bucket.sh
index 8d8bc97..a8937ea 100644
--- a/tests/commands/create_bucket.sh
+++ b/tests/commands/create_bucket.sh
@@ -36,7 +36,7 @@ create_bucket() {
     log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2"
     error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
-    error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
+    error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" --region "$AWS_REGION" 2>&1) || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
diff --git a/tests/commands/get_bucket_location.sh b/tests/commands/get_bucket_location.sh
index 5b77c12..411d9fd 100644
--- a/tests/commands/get_bucket_location.sh
+++ b/tests/commands/get_bucket_location.sh
@@ -26,7 +26,11 @@ get_bucket_location() {
   elif [[ $1 == 's3cmd' ]]; then
     get_bucket_location_s3cmd "$2" || get_result=$?
   elif [[ $1 == 'mc' ]]; then
-    get_bucket_location_mc "$2" || get_result=$?
+    if ! get_bucket_location_mc "$2"; then
+      log 2 "error getting mc bucket location"
+      return 1
+    fi
+    return 0
   else
     log 2 "command type '$1' not implemented for get_bucket_location"
     return 1
@@ -69,13 +73,11 @@ get_bucket_location_s3cmd() {
 
 get_bucket_location_mc() {
   record_command "get-bucket-location" "client:mc"
-  if [[ $# -ne 1 ]]; then
-    log 2 "get bucket location (mc) requires bucket name"
+  if ! check_param_count_v2 "bucket name" 1 $#; then
     return 1
   fi
-  info=$(send_command mc --insecure stat "$MC_ALIAS/$1") || results=$?
-  if [[ $results -ne 0 ]]; then
-    log 2 "error getting s3cmd info: $info"
+  if ! info=$(send_command mc --insecure stat "$MC_ALIAS/$1" 2>&1); then
+    log 2 "error getting mc info: $info"
     return 1
   fi
   # shellcheck disable=SC2034
diff --git a/tests/drivers/get_bucket_location/get_bucket_location.sh b/tests/drivers/get_bucket_location/get_bucket_location.sh
index 611b937..a56a913 100644
--- a/tests/drivers/get_bucket_location/get_bucket_location.sh
+++ b/tests/drivers/get_bucket_location/get_bucket_location.sh
@@ -23,8 +23,13 @@ get_check_bucket_location_various() {
     return 1
   fi
   # shellcheck disable=SC2154
-  if [[ $bucket_location != "null" ]] && [[ $bucket_location != "us-east-1" ]]; then
-    log 2 "wrong location: '$bucket_location'"
+  if [ "$AWS_REGION" == "us-east-1" ]; then
+    if [ "$bucket_location" != "null" ]; then
+      log 2 "expected 'null' for 'us-east-1' region, got : '$bucket_location'"
+      return 1
+    fi
+  elif [ "$AWS_REGION" != "$bucket_location" ]; then
+    log 2 "expected bucket location of '$AWS_REGION', got '$bucket_location'"
     return 1
   fi
   return 0
diff --git a/tests/env.sh b/tests/env.sh
index ffe1917..4f4ed57 100644
--- a/tests/env.sh
+++ b/tests/env.sh
@@ -281,7 +281,7 @@ check_user_vars() {
       exit 1
     fi
     IAM_PARAMS="--s3-iam-access $AWS_ACCESS_KEY_ID --s3-iam-secret $AWS_SECRET_ACCESS_KEY \
-      --s3-iam-region us-east-1 --s3-iam-bucket $USERS_BUCKET --s3-iam-endpoint $AWS_ENDPOINT_URL \
+      --s3-iam-region $AWS_REGION --s3-iam-bucket $USERS_BUCKET --s3-iam-endpoint $AWS_ENDPOINT_URL \
       --s3-iam-noverify"
     export IAM_PARAMS
     return 0
diff --git a/tests/generate_matrix.sh b/tests/generate_matrix.sh
new file mode 100755
index 0000000..72ee9ab
--- /dev/null
+++ b/tests/generate_matrix.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# generate github-actions matrix for system.yml
+
+source ./tests/drivers/params.sh
+
+set -euo pipefail
+
+files=()
+iam_types=()
+regions=()
+idx=0
+
+check_for_and_load_test_file_and_params() {
+  if ! check_param_count_v2 "file name" 1 $#; then
+    exit 1
+  fi
+  if grep -q '@test' "$1"; then
+    if [ $(( idx % 8 )) -eq 0 ]; then
+      iam="s3"
+    else
+      iam="folder"
+    fi
+    iam_types+=("$iam")
+    if [ $(( idx % 4 )) -eq 0 ]; then
+      region="us-west-1"
+    else
+      region="us-east-1"
+    fi
+    regions+=("$region")
+    files+=("$1")
+    idx=$((idx + 1))
+  fi
+}
+
+while IFS= read -r f; do
+  check_for_and_load_test_file_and_params "$f"
+done < <(find tests -name 'test_*.sh' | sort)
+
+files_json_arr=$(printf '%s\n' "${files[@]}"   | jq -R . | jq -s .)
+regions_json_arr=$(printf '%s\n' "${regions[@]}" | jq -R . | jq -s .)
+iam_types_json_arr=$(printf '%s\n' "${iam_types[@]}" | jq -R . | jq -s .)
+
+matrix_json=$(
+  jq -n \
+    --argjson files   "$files_json_arr" \
+    --argjson regions "$regions_json_arr" \
+    --argjson iam_types "$iam_types_json_arr" \
+    '
+    {
+      include:
+        [ range(0; ($files|length)) as $i
+          | [
+              {
+                desc: ("Run " + $files[$i] + ", non-static, " + $regions[$i] + " region, " + $iam_types[$i] + " IAM type"),
+                RUN_SET: $files[$i],
+                AWS_REGION: $regions[$i],
+                IAM_TYPE: $iam_types[$i],
+                BACKEND: "posix",
+                RECREATE_BUCKETS: "true",
+                DELETE_BUCKETS_AFTER_TEST: "true"
+              },
+              {
+                desc: ("Run " + $files[$i] + ", static, " + $regions[$i] + " region, " + $iam_types[$i] + " IAM type"),
+                RUN_SET: $files[$i],
+                AWS_REGION: $regions[$i],
+                IAM_TYPE: $iam_types[$i],
+                BACKEND: "posix",
+                RECREATE_BUCKETS: "false",
+                DELETE_BUCKETS_AFTER_TEST: "false"
+              }
+            ]
+        ] | add
+    }
+    '
+)
+
+echo "$matrix_json"
diff --git a/tests/run.sh b/tests/run.sh
index 36750b0..ff39e75 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -14,293 +14,88 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Function to display help information
+source ./tests/drivers/params.sh
+
 show_help() {
-    echo "Usage: $0 [option...]"
-    echo "   -h, --help                             Display this help message and exit"
-    echo "                                          Separate the below by comma"
-    echo "   s3api                                  Run all tests with s3api cli"
-    echo "   s3api-multipart                        Run multipart tests with s3api cli"
-    echo "   s3api-bucket                           Run bucket tests with s3api cli"
-    echo "   s3api-object                           Run object tests with s3api cli"
-    echo "   s3api-policy                           Run policy tests with s3api cli"
-    echo "   s3api-user                             Run user tests with s3api cli"
-    echo "   s3                                     Run tests with s3 cli"
-    echo "   s3cmd                                  Run tests with s3cmd utility"
-    echo "   s3cmd-user                             Run user tests with s3cmd utility"
-    echo "   s3cmd-non-user                         Run non-user tests with s3cmd utility"
-    echo "   s3cmd-file-count                       Run file count test with s3cmd utility"
-    echo "   mc                                     Run tests with mc utility"
-    echo "   mc-non-file-count                      Run non-file count tests with mc utility"
-    echo "   mc-file-count                          Run file count test with mc utility"
-    echo "   rest                                   Run tests with rest cli"
-    echo "   rest-base                              Run REST base tests"
-    echo "   rest-acl                               Run REST ACL tests"
-    echo "   rest-chunked                           Run REST chunked upload tests"
-    echo "   rest-checksum                          Run REST checksum tests"
-    echo "   rest-create-bucket                     Run REST create bucket tests"
-    echo "   rest-delete-bucket-ownership-controls  Run REST delete bucket ownership controls tests"
-    echo "   rest-delete-bucket-tagging             Run REST delete bucket tagging tests"
-    echo "   rest-get-bucket-location               Run REST get bucket location tests"
-    echo "   rest-get-object-tagging                Run REST get object tagging tests"
-    echo "   rest-head-bucket                       Run REST head bucket tests"
-    echo "   rest-list-buckets                      Run REST list-buckets tests"
-    echo "   rest-list-object-versions              Run REST list-object-versions tests"
-    echo "   rest-multipart                         Run REST multipart tests"
-    echo "   rest-not-implemented                   Run REST multipart tests"
-    echo "   rest-put-bucket-tagging                Run REST put-bucket-tagging tests"
-    echo "   rest-put-object                        Run REST put-object tests"
-    echo "   rest-put-object-tagging                Run REST put-object-tagging tests"
-    echo "   rest-versioning                        Run REST versioning tests"
-    echo "   rest-bucket                            Run REST bucket tests"
+  echo "Usage: $0 [option...]"
+  echo "   -h, --help                             Display this help message and exit"
+  echo "                                          Separate the below by comma"
+  echo "all                                       Attempt to run all tests (not recommended)"
+  echo_help_lines
+}
+
+echo_help_lines() {
+  gather_test_files
+
+  for run_set in "${run_sets[@]}"; do
+    description=${run_set/-/ }
+    if [[ "$description" == *"rest"* ]]; then
+      description=${description/rest/REST}
+    fi
+    spaces_needed=$((42-${#run_set}))
+    printf "%s%-${spaces_needed}s%s\n" "$run_set" "" "Run $description tests"
+  done
+}
+
+gather_test_files() {
+  while IFS= read -r f; do
+    if grep -q '@test' "$f"; then
+      files+=("$f")
+      file_without_header=${f/tests\/test_/}
+      file_without_sh=${file_without_header/.sh/}
+      run_set=${file_without_sh//_/-}
+      run_sets+=("$run_set")
+    fi
+  done < <(find tests -name 'test_*.sh' | sort)
+}
+
+run_set_if_matching() {
+  if ! check_param_count_v2 "set name" 1 $#; then
+    exit 1
+  fi
+  if [ "$1" == "all" ]; then
+    echo "running '${run_sets[$idx]}' test suite ..."
+    if ! "$HOME"/bin/bats "${files[$idx]}"; then
+      echo "error running '${files[$idx]}' tests"
+      exit 1
+    fi
+    suite_run="true"
+  elif [ "$run_set" == "$1" ]; then
+    echo "running '${run_sets[$idx]}' test suite ..."
+    if ! "$HOME"/bin/bats "${files[$idx]}"; then
+      echo "error running '${files[$idx]}' tests"
+      exit 1
+    fi
+    complete="true"
+    suite_run="true"
+  fi
 }
 
 handle_param() {
-  case $1 in
-      -h|--help)
-          show_help
-          exit 0
-          ;;
-      s3|s3-file-count|s3-non-file-count|s3api|s3cmd|s3cmd-user|s3cmd-non-user|\
-      s3cmd-file-count|mc|mc-non-file-count|mc-file-count|s3api-user|rest|s3api-policy|\
-      s3api-bucket|s3api-object|s3api-multipart|rest-base|rest-acl|rest-chunked|rest-checksum|\
-      rest-create-bucket|rest-head-bucket|rest-list-buckets|rest-not-implemented|\
-      rest-put-object|rest-versioning|rest-bucket|rest-multipart|rest-delete-bucket-ownership-controls|\
-      rest-delete-bucket-tagging|setup-remove-static|rest-put-bucket-tagging|rest-get-bucket-location|\
-      rest-put-object-tagging|rest-get-object-tagging|rest-list-object-versions|rest-get-bucket-policy-status)
-          run_suite "$1"
-          ;;
-      *) # Handle unrecognized options or positional arguments
-          echo "Unrecognized option: $1" >&2
-          exit 1
-          ;;
-  esac
-}
+  if ! check_param_count_v2 "run sets, separated by comma" 1 $#; then
+    exit 1
+  fi
 
-run_suite() {
-  exit_code=0
-  case $1 in
-    s3api)
-      echo "Running all s3api tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$?
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$?
-      fi
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$?
-      fi
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$?
-      fi
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$?
-      fi
-      ;;
-    s3api-multipart)
-      echo "Running s3api multipart tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$?
-      ;;
-    s3api-policy)
-      echo "Running s3api policy tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$?
-      ;;
-    s3api-bucket)
-      echo "Running s3api bucket tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$?
-      ;;
-    s3api-object)
-      echo "Running s3api object tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$?
-      ;;
-    s3)
-      echo "Running s3 tests ..."
-      "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$?
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$?
-      fi
-      ;;
-    s3-non-file-count)
-      echo "Running s3 non-file count tests ..."
-      "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$?
-      ;;
-    s3-file-count)
-      echo "Running s3 file count test ..."
-      "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$?
-      ;;
-    s3cmd)
-      echo "Running s3cmd tests ..."
-      "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$?
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$?
-      fi
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$?
-      fi
-      ;;
-    s3cmd-user)
-      echo "Running s3cmd user tests ..."
-      "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$?
-      ;;
-    s3cmd-non-user)
-      echo "Running s3cmd non-user tests ..."
-      "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$?
-      ;;
-    s3cmd-file-count)
-      echo "Running s3cmd file count test ..."
-      "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$?
-      ;;
-    mc)
-      echo "Running mc tests ..."
-      "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$?
-      fi
-      ;;
-    mc-non-file-count)
-      echo "Running mc non-file count tests ..."
-      "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
-      ;;
-    mc-file-count)
-      echo "Running mc file count test ..."
-      "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$?
-      ;;
-    rest)
-      echo "Running rest tests ..."
-      if ! "$HOME"/bin/bats ./tests/test_rest.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_acl.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_chunked.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_checksum.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_create_bucket.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_delete_bucket_ownership_controls.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_get_bucket_location.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_get_bucket_policy_status.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_get_object_tagging.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_head_bucket.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_list_buckets.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_list_object_versions.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_multipart.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_not_implemented.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_put_bucket_tagging.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_put_object.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_put_object_tagging.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_versioning.sh; then
-        exit_code=1
-      elif ! "$HOME"/bin/bats ./tests/test_rest_bucket.sh; then
-        exit_code=1
-      fi
-      ;;
-    rest-base)
-      echo "Running REST base tests ..."
-      "$HOME"/bin/bats ./tests/test_rest.sh || exit_code=$?
-      ;;
-    rest-acl)
-      echo "Running REST ACL tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_acl.sh || exit_code=$?
-      ;;
-    rest-bucket)
-      echo "Running REST bucket tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_bucket.sh || exit_code=$?
-      ;;
-    rest-chunked)
-      echo "Running REST chunked upload tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_chunked.sh || exit_code=$?
-      ;;
-    rest-checksum)
-      echo "Running REST checksum tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_checksum.sh || exit_code=$?
-      ;;
-    rest-create-bucket)
-      echo "Running REST create bucket tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_create_bucket.sh || exit_code=$?
-      ;;
-    rest-delete-bucket-ownership-controls)
-      echo "Running REST delete bucket ownership controls tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_delete_bucket_ownership_controls.sh || exit_code=$?
-      ;;
-    rest-delete-bucket-tagging)
-      echo "Running REST delete bucket tagging tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_delete_bucket_tagging.sh || exit_code=$?
-      ;;
-    rest-get-bucket-location)
-      echo "Running REST get bucket location tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_get_bucket_location.sh || exit_code=$?
-      ;;
-    rest-get-bucket-policy-status)
-      echo "Running REST get bucket policy status tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_get_bucket_policy_status.sh || exit_code=$?
-      ;;
-    rest-get-object-tagging)
-      echo "Running REST get object tagging tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_get_object_tagging.sh || exit_code=$?
-      ;;
-    rest-head-bucket)
-      echo "Running REST head bucket tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_head_bucket.sh || exit_code=$?
-      ;;
-    rest-list-buckets)
-      echo "Running REST list-buckets tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_list_buckets.sh || exit_code=$?
-      ;;
-    rest-list-object-versions)
-      echo "Running REST list-object-versions tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_list_object_versions.sh || exit_code=$?
-      ;;
-    rest-not-implemented)
-      echo "Running REST not-implemented tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_not_implemented.sh || exit_code=$?
-      ;;
-    rest-multipart)
-      echo "Running REST multipart tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_multipart.sh || exit_code=$?
-      ;;
-    rest-put-bucket-tagging)
-      echo "Running REST put-bucket-tagging tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_put_bucket_tagging.sh || exit_code=$?
-      ;;
-    rest-put-object)
-      echo "Running REST put-object tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_put_object.sh || exit_code=$?
-      ;;
-    rest-put-object-tagging)
-      echo "Running REST put-object-tagging tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_put_object_tagging.sh || exit_code=$?
-      ;;
-    rest-versioning)
-      echo "Running REST versioning tests ..."
-      "$HOME"/bin/bats ./tests/test_rest_versioning.sh || exit_code=$?
-      ;;
-    s3api-user)
-      echo "Running s3api user tests ..."
-      "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$?
-      ;;
-    setup-remove-static)
-      echo "Testing setup/remove static bucket scripts ..."
-      VERSITYGW_TEST_ENV="$VERSITYGW_TEST_ENV" ./tests/test_setup_remove_static.sh || exit_code=$?
-      ;;
-  esac
-  if [ "$exit_code" -ne 0 ]; then
+  gather_test_files
+
+  idx=0
+  complete="false"
+  suite_run="false"
+  for run_set in "${run_sets[@]}"; do
+    run_set_if_matching "$1"
+    if [ "$complete" == "true" ]; then
+      break
+    fi
+    ((idx++))
+  done
+
+  if [ "$suite_run" == "false" ]; then
+    echo "no suites matching '$1'"
     exit 1
   fi
 }
 
-if [ $# -le 0 ]; then
+if [ $# -le 0 ] || [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
   show_help
   exit 0
 fi
diff --git a/tests/run_all.sh b/tests/run_all.sh
index 45ead44..2c19017 100755
--- a/tests/run_all.sh
+++ b/tests/run_all.sh
@@ -24,10 +24,8 @@ grep -v ^# "$VERSITYGW_TEST_ENV"
 
 status=0
 
-for cmd in s3api s3 s3cmd mc rest; do
-  if ! ./tests/run.sh "$cmd"; then
-    status=1
-  fi
-done
+if ! ./tests/run.sh "all"; then
+  status=1
+fi
 
 exit $status
diff --git a/tests/test_rest_delete_bucket.sh b/tests/test_rest_delete_bucket.sh
index 4b68e72..b5c68cb 100755
--- a/tests/test_rest_delete_bucket.sh
+++ b/tests/test_rest_delete_bucket.sh
@@ -23,6 +23,9 @@ source ./tests/drivers/delete_object/delete_object_rest.sh
 source ./tests/drivers/put_object/put_object_rest.sh
 
 @test "REST - DeleteBucket - can delete with partial multipart upload" {
+  if [ "$RECREATE_BUCKETS" == "false" ]; then
+    skip "avoid bucket deletion in static mode"
+  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name="$output"
@@ -38,6 +41,9 @@ source ./tests/drivers/put_object/put_object_rest.sh
 }
 
 @test "REST - DeleteBucket - file - non-versioning" {
+  if [ "$RECREATE_BUCKETS" == "false" ]; then
+    skip "avoid bucket deletion in static mode"
+  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name="$output"
@@ -56,6 +62,9 @@ source ./tests/drivers/put_object/put_object_rest.sh
 }
 
 @test "REST - DeleteBucket - file - versioning" {
+  if [ "$RECREATE_BUCKETS" == "false" ]; then
+    skip "avoid bucket deletion in static mode"
+  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name="$output"
@@ -83,6 +92,9 @@ source ./tests/drivers/put_object/put_object_rest.sh
 }
 
 @test "REST - DeleteBucket - invalid x-amz-expected-bucket-owner" {
+  if [ "$RECREATE_BUCKETS" == "false" ]; then
+    skip "avoid bucket deletion in static mode"
+  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name="$output"
@@ -105,6 +117,9 @@ source ./tests/drivers/put_object/put_object_rest.sh
 }
 
 @test "REST - DeleteBucket - incorrect x-amz-expected-bucket-owner" {
+  if [ "$RECREATE_BUCKETS" == "false" ]; then
+    skip "avoid bucket deletion in static mode"
+  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name="$output"
@@ -117,6 +132,9 @@ source ./tests/drivers/put_object/put_object_rest.sh
 }
 
 @test "REST - DeleteBucket - correct x-amz-expected-bucket-owner" {
+  if [ "$RECREATE_BUCKETS" == "false" ]; then
+    skip "avoid bucket deletion in static mode"
+  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name="$output"
diff --git a/tests/test_rest_put_bucket_policy.sh b/tests/test_rest_put_bucket_policy.sh
index 7741c7e..b7de9e7 100755
--- a/tests/test_rest_put_bucket_policy.sh
+++ b/tests/test_rest_put_bucket_policy.sh
@@ -22,6 +22,9 @@ source ./tests/drivers/put_bucket_policy/put_bucket_policy_rest.sh
 source ./tests/setup.sh
 
 @test "PutBucketPolicy - success returns 204" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1712"
+  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name="$output"
diff --git a/tests/test_s3api_bucket.sh b/tests/test_s3api_bucket.sh
index f65c727..afb314f 100755
--- a/tests/test_s3api_bucket.sh
+++ b/tests/test_s3api_bucket.sh
@@ -96,6 +96,9 @@ export RUN_USERS=true
 
 # get-bucket-location
 @test "test_get_bucket_location" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1643"
+  fi
   test_common_get_bucket_location "s3api"
 }
 
diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh
index a6b06b9..8bac003 100755
--- a/tests/test_s3cmd.sh
+++ b/tests/test_s3cmd.sh
@@ -74,6 +74,9 @@ export RUN_USERS=true
 
 # get-bucket-location
 @test "test_get_bucket_location" {
+  if [ "$DIRECT" != "true" ]; then
+      skip "https://github.com/versity/versitygw/issues/1643"
+    fi
   test_common_get_bucket_location "s3cmd"
 }
 
diff --git a/tests/util/util_users.sh b/tests/util/util_users.sh
index c4901c8..e08cfe1 100644
--- a/tests/util/util_users.sh
+++ b/tests/util/util_users.sh
@@ -242,7 +242,7 @@ create_user_with_user() {
     log 2 "create user with user command requires creator ID, key, and new user ID, key, and role"
     return 1
   fi
-  if ! error=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" create-user --access "$3" --secret "$4" --role "$5" 2>&1); then
+  if ! error=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$1" --secret "$2" --region "$AWS_REGION" --endpoint-url "$AWS_ENDPOINT_URL" create-user --access "$3" --secret "$4" --role "$5" 2>&1); then
     log 2 "error creating user: $error"
     return 1
   fi
@@ -362,8 +362,7 @@ delete_user_versitygw() {
     log 2 "delete user via versitygw command requires user ID or username"
     return 1
   fi
-  log 5 "$VERSITY_EXE admin --allow-insecure --access $AWS_ACCESS_KEY_ID --secret $AWS_SECRET_ACCESS_KEY --endpoint-url $AWS_ENDPOINT_URL delete-user --access $1"
-  if ! error=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" delete-user --access "$1" 2>&1); then
+  if ! error=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --region "$AWS_REGION" --endpoint-url "$AWS_ENDPOINT_URL" delete-user --access "$1" 2>&1); then
     log 2 "error deleting user: $error"
     export error
     return 1
@@ -413,7 +412,7 @@ change_bucket_owner() {
     skip "any direct commands requiring bucket ownership change not yet implemented"
   fi
   log 5 "changing owner for bucket $3, new owner: $4"
-  error=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" change-bucket-owner --bucket "$3" --owner "$4" 2>&1) || local change_result=$?
+  error=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$1" --secret "$2" --region "$AWS_REGION" --endpoint-url "$AWS_ENDPOINT_URL" change-bucket-owner --bucket "$3" --owner "$4" 2>&1) || local change_result=$?
   if [[ $change_result -ne 0 ]]; then
     log 2 "error changing bucket owner: $error"
     return 1
@@ -427,7 +426,7 @@ get_bucket_owner() {
     log 2 "'get bucket owner' command requires bucket name"
     return 1
   fi
-  if ! buckets=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-buckets 2>&1); then
+  if ! buckets=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --region "$AWS_REGION" --endpoint-url "$AWS_ENDPOINT_URL" list-buckets 2>&1); then
     log 2 "error listing buckets: $buckets"
     return 1
   fi
diff --git a/tests/versity.sh b/tests/versity.sh
index f0fbcbe..5a7d938 100644
--- a/tests/versity.sh
+++ b/tests/versity.sh
@@ -114,7 +114,7 @@ run_versity_app_s3() {
   if ! check_param_count "run_versity_app_s3" "versityid app index" 1 $#; then
     exit 1
   fi
-  base_command=("$VERSITY_EXE" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY")
+  base_command=("$VERSITY_EXE" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY" --region="$AWS_REGION")
   if [ -n "$CERT" ] && [ -n "$KEY" ]; then
     base_command+=(--cert "$CERT" --key "$KEY")
   fi