test: github-actions speedup, cleanup

.github/workflows/system.yml

@@ -8,138 +8,96 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - set: "s3cmd, posix"
-            LOCAL_FOLDER: /tmp/gw1
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
+          - set: "mc, posix, non-file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam1
-            AWS_ENDPOINT_URL: https://127.0.0.1:7070
-            RUN_SET: "s3cmd"
+            RUN_SET: "mc-non-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7070
             BACKEND: "posix"
-          - set: "s3, posix"
-            LOCAL_FOLDER: /tmp/gw2
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
+          - set: "mc, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam2
-            AWS_ENDPOINT_URL: https://127.0.0.1:7071
-            RUN_SET: "s3"
+            RUN_SET: "mc-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7071
             BACKEND: "posix"
-          - set: "s3api non-policy, posix"
-            LOCAL_FOLDER: /tmp/gw3
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
+          - set: "REST, posix, non-static, all, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam3
-            AWS_ENDPOINT_URL: https://127.0.0.1:7072
-            RUN_SET: "s3api"
-            RECREATE_BUCKETS: "true"
-            PORT: 7072
-            BACKEND: "posix"
-          - set: "mc, posix"
-            LOCAL_FOLDER: /tmp/gw4
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam4
-            AWS_ENDPOINT_URL: https://127.0.0.1:7073
-            RUN_SET: "mc"
-            RECREATE_BUCKETS: "true"
-            PORT: 7073
-            BACKEND: "posix"
-          - set: "s3api-user, posix, s3 IAM"
-            LOCAL_FOLDER: /tmp/gw5
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
-            IAM_TYPE: s3
-            USERS_BUCKET: versity-gwtest-iam
-            AWS_ENDPOINT_URL: https://127.0.0.1:7074
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "true"
-            PORT: 7074
-            BACKEND: "posix"
-          - set: "s3api non-policy, static buckets"
-            LOCAL_FOLDER: /tmp/gw6
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam6
-            AWS_ENDPOINT_URL: https://127.0.0.1:7075
-            RUN_SET: "s3api-non-policy"
-            RECREATE_BUCKETS: "false"
-            PORT: 7075
-            BACKEND: "posix"
-          - set: "s3api non-policy, s3 backend"
-            LOCAL_FOLDER: /tmp/gw7
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam7
-            AWS_ENDPOINT_URL: https://127.0.0.1:7076
-            RUN_SET: "s3api"
-            RECREATE_BUCKETS: "true"
-            PORT: 7076
-            BACKEND: "s3"
-          - set: "REST, posix"
-            LOCAL_FOLDER: /tmp/gw8
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam8
-            AWS_ENDPOINT_URL: https://127.0.0.1:7077
             RUN_SET: "rest"
             RECREATE_BUCKETS: "true"
-            PORT: 7077
             BACKEND: "posix"
-          - set: "s3api policy, static buckets"
-            LOCAL_FOLDER: /tmp/gw9
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-8
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-8
+          - set: "s3, posix, non-file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-non-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, non-static, s3 IAM"
+            IAM_TYPE: s3
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, multipart, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-multipart"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, object, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-object"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam9
-            AWS_ENDPOINT_URL: https://127.0.0.1:7078
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "false"
-            PORT: 7078
             BACKEND: "posix"
-          - set: "s3api user, static buckets"
-            LOCAL_FOLDER: /tmp/gw10
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-9
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-9
+          - set: "s3api, posix, user, static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam10
-            AWS_ENDPOINT_URL: https://127.0.0.1:7079
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "false"
-            PORT: 7079
             BACKEND: "posix"
-          - set: "s3api policy and user, posix"
-            LOCAL_FOLDER: /tmp/gw11
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-10
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-10
+          - set: "s3api, s3, multipart|object, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam11
-            AWS_ENDPOINT_URL: https://127.0.0.1:7080
-            RUN_SET: "s3api-policy,s3api-user"
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
             RECREATE_BUCKETS: "true"
-            PORT: 7080
-            BACKEND: "posix"
-          - set: "s3api policy and user, s3 backend"
-            LOCAL_FOLDER: /tmp/gw12
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-11
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-11
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam12
-            AWS_ENDPOINT_URL: https://127.0.0.1:7081
-            RUN_SET: "s3api-policy,s3api-user"
-            RECREATE_BUCKETS: "true"
-            PORT: 7081
             BACKEND: "s3"
+          - set: "s3api, s3, policy|user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy,s3api-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "s3"
+          - set: "s3cmd, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, non-user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-non-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
@@ -176,15 +134,8 @@ jobs:
 
       - name: Build and run
         env:
-          LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
-          BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
-          BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
-          USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
-          USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
           IAM_TYPE: ${{ matrix.IAM_TYPE }}
-          AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
           RUN_SET: ${{ matrix.RUN_SET }}
-          PORT: ${{ matrix.PORT }}
           AWS_PROFILE: versity
           VERSITY_EXE: ${{ github.workspace }}/versitygw
           RUN_VERSITYGW: true
@@ -192,6 +143,13 @@ jobs:
           RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
+          LOCAL_FOLDER: /tmp/gw
+          BUCKET_ONE_NAME: versity-gwtest-bucket-one
+          BUCKET_TWO_NAME: versity-gwtest-bucket-two
+          USERS_FOLDER: /tmp/iam
+          USERS_BUCKET: versity-gwtest-iam
+          AWS_ENDPOINT_URL: https://127.0.0.1:7070
+          PORT: 7070
           S3CMD_CONFIG: tests/s3cfg.local.default
           MC_ALIAS: versity
           LOG_LEVEL: 4
@@ -204,6 +162,7 @@ jobs:
           REMOVE_TEST_FILE_FOLDER: true
           VERSIONING_DIR: ${{ github.workspace }}/versioning
           COMMAND_LOG: command.log
+          TIME_LOG: time.log
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -224,6 +183,9 @@ jobs:
           fi
           BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
+      - name: Time report
+        run: cat ${{ github.workspace }}/time.log
+
       - name: Coverage report
         run: |
           go tool covdata percent -i=cover

tests/.env.default

@@ -28,4 +28,5 @@ PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 REMOVE_TEST_FILE_FOLDER=true
 VERSIONING_DIR=/tmp/versioning
-COMMAND_LOG=command.log
+COMMAND_LOG=command.log
+TIME_LOG=time.log

tests/docker-compose-bats.yml

@@ -27,7 +27,7 @@ services:
       args:
         - CONFIG_FILE=tests/.env.default
     image: bats_test
-    command: ["s3api-non-policy"]
+    command: ["s3api-bucket,s3api-object"]
   direct:
     build:
       dockerfile: tests/Dockerfile_direct

tests/run.sh

@@ -19,14 +19,21 @@ show_help() {
     echo "Usage: $0 [option...]"
     echo "   -h, --help          Display this help message and exit"
     echo "                       Separate the below by comma"
-    echo "   s3api               Run tests with s3api cli"
-    echo "   s3api-non-policy    Run policy tests with s3api cli"
+    echo "   s3api               Run all tests with s3api cli"
+    echo "   s3api-multipart     Run multipart tests with s3api cli"
+    echo "   s3api-bucket        Run bucket tests with s3api cli"
+    echo "   s3api-object        Run object tests with s3api cli"
     echo "   s3api-policy        Run policy tests with s3api cli"
+    echo "   s3api-user          Run user tests with s3api cli"
     echo "   s3                  Run tests with s3 cli"
     echo "   s3cmd               Run tests with s3cmd utility"
+    echo "   s3cmd-user          Run user tests with s3cmd utility"
+    echo "   s3cmd-non-user      Run non-user tests with s3cmd utility"
+    echo "   s3cmd-file-count    Run file count test with s3cmd utility"
     echo "   mc                  Run tests with mc utility"
+    echo "   mc-non-file-count   Run non-file count tests with mc utility"
+    echo "   mc-file-count       Run file count test with mc utility"
     echo "   rest                Run tests with rest cli"
-    echo "   s3api-user          Run user tests with aws cli"
 }
 
 handle_param() {
@@ -35,7 +42,7 @@ handle_param() {
           show_help
           exit 0
           ;;
-      s3|s3api|s3cmd|mc|s3api-user|rest|s3api-policy|s3api-non-policy)
+      s3|s3-file-count|s3-non-file-count|s3api|s3cmd|s3cmd-user|s3cmd-non-user|s3cmd-file-count|mc|mc-non-file-count|mc-file-count|s3api-user|rest|s3api-policy|s3api-bucket|s3api-object|s3api-multipart)
           run_suite "$1"
           ;;
       *) # Handle unrecognized options or positional arguments
@@ -50,25 +57,50 @@ run_suite() {
   case $1 in
     s3api)
       echo "Running all s3api tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api.sh || exit_code=$?
+      "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$?
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$?
+      fi
       if [[ $exit_code -eq 0 ]]; then
         "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$?
       fi
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$?
+      fi
       if [[ $exit_code -eq 0 ]]; then
         "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$?
       fi
       ;;
+    s3api-multipart)
+      echo "Running s3api multipart tests ..."
+      "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$?
+      ;;
     s3api-policy)
       echo "Running s3api policy tests ..."
       "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$?
       ;;
-    s3api-non-policy)
-      echo "Running s3api non-policy tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api.sh || exit_code=$?
+    s3api-bucket)
+      echo "Running s3api bucket tests ..."
+      "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$?
+      ;;
+    s3api-object)
+      echo "Running s3api object tests ..."
+      "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$?
       ;;
     s3)
       echo "Running s3 tests ..."
       "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$?
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$?
+      fi
+      ;;
+    s3-non-file-count)
+      echo "Running s3 non-file count tests ..."
+      "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$?
+      ;;
+    s3-file-count)
+      echo "Running s3 file count test ..."
+      "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$?
       ;;
     s3cmd)
       echo "Running s3cmd tests ..."
@@ -76,10 +108,36 @@ run_suite() {
       if [[ $exit_code -eq 0 ]]; then
         "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$?
       fi
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$?
+      fi
+      ;;
+    s3cmd-user)
+      echo "Running s3cmd user tests ..."
+      "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$?
+      ;;
+    s3cmd-non-user)
+      echo "Running s3cmd non-user tests ..."
+      "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$?
+      ;;
+    s3cmd-file-count)
+      echo "Running s3cmd file count test ..."
+      "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$?
       ;;
     mc)
       echo "Running mc tests ..."
       "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$?
+      fi
+      ;;
+    mc-non-file-count)
+      echo "Running mc non-file count tests ..."
+      "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
+      ;;
+    mc-file-count)
+      echo "Running mc file count test ..."
+      "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$?
       ;;
     rest)
       echo "Running rest tests ..."

tests/setup.sh

@@ -28,7 +28,7 @@ setup() {
   base_setup
 
   log 4 "Running test $BATS_TEST_NAME"
-  if [[ $LOG_LEVEL -ge 5 ]]; then
+  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
     start_time=$(date +%s)
     export start_time
   fi
@@ -75,9 +75,13 @@ teardown() {
     fi
   fi
   stop_versity
-  if [[ $LOG_LEVEL -ge 5 ]]; then
+  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
     end_time=$(date +%s)
-    log 4 "Total test time: $((end_time - start_time))"
+    total_time=$((end_time - start_time))
+    log 4 "Total test time: $total_time"
+    if [[ -n "$TIME_LOG" ]]; then
+      echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
+    fi
   fi
   if [[ -n "$COVERAGE_DB" ]]; then
     record_result

tests/test_mc.sh

@@ -112,10 +112,6 @@ export RUN_MC=true
   test_common_presigned_url_utf8_chars "mc"
 }
 
-@test "test_list_objects_file_count" {
-  test_common_list_objects_file_count "mc"
-}
-
 @test "test_create_bucket_invalid_name_mc" {
   if [[ $RECREATE_BUCKETS != "true" ]]; then
     return

tests/test_mc_file_count.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/test_common.sh
+
+export RUN_MC=true
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "mc"
+}

tests/test_s3.sh

@@ -51,10 +51,6 @@ source ./tests/util_file.sh
   test_common_list_buckets "s3"
 }
 
-@test "test_list_objects_file_count" {
-  test_common_list_objects_file_count "s3"
-}
-
 @test "test_delete_bucket" {
   if [[ $RECREATE_BUCKETS == "false" ]]; then
     skip "will not test bucket deletion in static bucket test config"

tests/test_s3_file_count.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/test_common.sh
+source ./tests/util_file.sh
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "s3"
+}

tests/test_s3api_bucket.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/setup.sh
+source ./tests/util.sh
+source ./tests/util_aws.sh
+source ./tests/util_create_bucket.sh
+source ./tests/util_file.sh
+source ./tests/util_lock_config.sh
+source ./tests/util_tags.sh
+source ./tests/util_users.sh
+source ./tests/test_aws_root_inner.sh
+source ./tests/test_common.sh
+source ./tests/test_common_acl.sh
+source ./tests/commands/copy_object.sh
+source ./tests/commands/delete_bucket_policy.sh
+source ./tests/commands/delete_object_tagging.sh
+source ./tests/commands/get_bucket_acl.sh
+source ./tests/commands/get_bucket_policy.sh
+source ./tests/commands/get_bucket_versioning.sh
+source ./tests/commands/get_object.sh
+source ./tests/commands/get_object_attributes.sh
+source ./tests/commands/get_object_legal_hold.sh
+source ./tests/commands/get_object_lock_configuration.sh
+source ./tests/commands/get_object_retention.sh
+source ./tests/commands/get_object_tagging.sh
+source ./tests/commands/list_object_versions.sh
+source ./tests/commands/put_bucket_acl.sh
+source ./tests/commands/put_bucket_policy.sh
+source ./tests/commands/put_bucket_versioning.sh
+source ./tests/commands/put_object.sh
+source ./tests/commands/put_object_legal_hold.sh
+source ./tests/commands/put_object_lock_configuration.sh
+source ./tests/commands/put_object_retention.sh
+source ./tests/commands/put_public_access_block.sh
+source ./tests/commands/select_object_content.sh
+
+export RUN_USERS=true
+
+# create-bucket
+@test "test_create_delete_bucket_aws" {
+  test_common_create_delete_bucket "aws"
+}
+
+@test "test_create_bucket_invalid_name" {
+  test_create_bucket_invalid_name_aws_root
+}
+
+# delete-bucket - test_create_delete_bucket_aws
+
+# delete-bucket-policy
+@test "test_get_put_delete_bucket_policy" {
+  if [[ -n $SKIP_POLICY ]]; then
+    skip "will not test policy actions with SKIP_POLICY set"
+  fi
+  test_common_get_put_delete_bucket_policy "aws"
+}
+
+# delete-bucket-tagging
+@test "test-set-get-delete-bucket-tags" {
+  test_common_set_get_delete_bucket_tags "aws"
+}
+
+# get-bucket-acl
+@test "test_get_bucket_acl" {
+  test_get_bucket_acl_aws_root
+}
+
+# get-bucket-location
+@test "test_get_bucket_location" {
+  test_common_get_bucket_location "aws"
+}
+
+# get-bucket-policy - test_get_put_delete_bucket_policy
+
+# get-bucket-tagging - test_set_get_delete_bucket_tags
+
+@test "test_head_bucket_invalid_name" {
+  if head_bucket "aws" ""; then
+    fail "able to get bucket info for invalid name"
+  fi
+}
+
+# test listing buckets on versitygw
+@test "test_list_buckets" {
+  test_common_list_buckets "s3api"
+}
+
+@test "test_put_bucket_acl" {
+  test_common_put_bucket_acl "s3api"
+}
+
+@test "test_head_bucket" {
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info"
+  log 5 "INFO:  $bucket_info"
+  region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region"
+  [[ $region != "" ]] || fail "empty bucket region"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_head_bucket_doesnt_exist" {
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+}

tests/test_s3api_multipart.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/setup.sh
+source ./tests/test_aws_root_inner.sh
+source ./tests/util_file.sh
+source ./tests/util_multipart.sh
+source ./tests/util_tags.sh
+source ./tests/commands/get_object.sh
+source ./tests/commands/put_object.sh
+source ./tests/commands/list_multipart_uploads.sh
+
+# abort-multipart-upload
+@test "test_abort_multipart_upload" {
+  test_abort_multipart_upload_aws_root
+}
+
+# complete-multipart-upload
+@test "test_complete_multipart_upload" {
+  test_complete_multipart_upload_aws_root
+}
+
+# create-multipart-upload
+@test "test_create_multipart_upload_properties" {
+  test_create_multipart_upload_properties_aws_root
+}
+
+# test multi-part upload list parts command
+@test "test-multipart-upload-list-parts" {
+  test_multipart_upload_list_parts_aws_root
+}
+
+# test listing of active uploads
+@test "test-multipart-upload-list-uploads" {
+  local bucket_file_one="bucket-file-one"
+  local bucket_file_two="bucket-file-two"
+
+  if [[ $RECREATE_BUCKETS == false ]]; then
+    run abort_all_multipart_uploads "$BUCKET_ONE_NAME"
+    assert_success
+  fi
+
+  run create_test_files "$bucket_file_one" "$bucket_file_two"
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two"
+  assert_success
+}
+
+@test "test-multipart-upload-from-bucket" {
+  local bucket_file="bucket-file"
+
+  run create_test_file "$bucket_file"
+  assert_success
+
+  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
+  assert_success
+
+  run get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy"
+  assert_success
+
+  run compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file
+  assert_success
+}
+
+@test "test_multipart_upload_from_bucket_range_too_large" {
+  local bucket_file="bucket-file"
+  run create_large_file "$bucket_file"
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file"
+  assert_success
+}
+
+@test "test_multipart_upload_from_bucket_range_valid" {
+  local bucket_file="bucket-file"
+  run create_large_file "$bucket_file"
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file"
+  assert_success
+}

tests/test_s3api_object.sh

@@ -20,7 +20,6 @@ source ./tests/util_aws.sh
 source ./tests/util_create_bucket.sh
 source ./tests/util_file.sh
 source ./tests/util_lock_config.sh
-source ./tests/util_multipart.sh
 source ./tests/util_tags.sh
 source ./tests/util_users.sh
 source ./tests/test_aws_root_inner.sh
@@ -38,7 +37,6 @@ source ./tests/commands/get_object_legal_hold.sh
 source ./tests/commands/get_object_lock_configuration.sh
 source ./tests/commands/get_object_retention.sh
 source ./tests/commands/get_object_tagging.sh
-source ./tests/commands/list_multipart_uploads.sh
 source ./tests/commands/list_object_versions.sh
 source ./tests/commands/put_bucket_acl.sh
 source ./tests/commands/put_bucket_policy.sh
@@ -52,16 +50,6 @@ source ./tests/commands/select_object_content.sh
 
 export RUN_USERS=true
 
-# abort-multipart-upload
-@test "test_abort_multipart_upload" {
-  test_abort_multipart_upload_aws_root
-}
-
-# complete-multipart-upload
-@test "test_complete_multipart_upload" {
-  test_complete_multipart_upload_aws_root
-}
-
 # copy-object
 @test "test_copy_object" {
   test_common_copy_object "s3api"
@@ -71,35 +59,6 @@ export RUN_USERS=true
   copy_object_empty || fail "copy objects with no parameters test failure"
 }
 
-# create-bucket
-@test "test_create_delete_bucket_aws" {
-  test_common_create_delete_bucket "aws"
-}
-
-@test "test_create_bucket_invalid_name" {
-  test_create_bucket_invalid_name_aws_root
-}
-
-# create-multipart-upload
-@test "test_create_multipart_upload_properties" {
-  test_create_multipart_upload_properties_aws_root
-}
-
-# delete-bucket - test_create_delete_bucket_aws
-
-# delete-bucket-policy
-@test "test_get_put_delete_bucket_policy" {
-  if [[ -n $SKIP_POLICY ]]; then
-    skip "will not test policy actions with SKIP_POLICY set"
-  fi
-  test_common_get_put_delete_bucket_policy "aws"
-}
-
-# delete-bucket-tagging
-@test "test-set-get-delete-bucket-tags" {
-  test_common_set_get_delete_bucket_tags "aws"
-}
-
 # delete-object - tested with bucket cleanup before or after tests
 
 # delete-object-tagging
@@ -115,20 +74,6 @@ export RUN_USERS=true
   test_delete_objects_aws_root
 }
 
-# get-bucket-acl
-@test "test_get_bucket_acl" {
-  test_get_bucket_acl_aws_root
-}
-
-# get-bucket-location
-@test "test_get_bucket_location" {
-  test_common_get_bucket_location "aws"
-}
-
-# get-bucket-policy - test_get_put_delete_bucket_policy
-
-# get-bucket-tagging - test_set_get_delete_bucket_tags
-
 # get-object
 @test "test_get_object_full_range" {
   test_get_object_full_range_aws_root
@@ -143,12 +88,6 @@ export RUN_USERS=true
   test_get_object_attributes_aws_root
 }
 
-@test "test_head_bucket_invalid_name" {
-  if head_bucket "aws" ""; then
-    fail "able to get bucket info for invalid name"
-  fi
-}
-
 @test "test_put_object" {
   test_put_object_aws_root
 }
@@ -168,11 +107,6 @@ export RUN_USERS=true
   test_common_put_object_no_data "aws"
 }
 
-# test listing buckets on versitygw
-@test "test_list_buckets" {
-  test_common_list_buckets "s3api"
-}
-
 # test listing a bucket's objects on versitygw
 @test "test_list_objects" {
   test_common_list_objects "aws"
@@ -186,10 +120,6 @@ export RUN_USERS=true
   test_get_put_object_retention_aws_root
 }
 
-@test "test_put_bucket_acl" {
-  test_common_put_bucket_acl "s3api"
-}
-
 # test v1 s3api list objects command
 @test "test-s3api-list-objects-v1" {
   test_s3api_list_objects_v1_aws_root
@@ -205,77 +135,6 @@ export RUN_USERS=true
   test_common_set_get_object_tags "aws"
 }
 
-# test multi-part upload list parts command
-@test "test-multipart-upload-list-parts" {
-  test_multipart_upload_list_parts_aws_root
-}
-
-# test listing of active uploads
-@test "test-multipart-upload-list-uploads" {
-  local bucket_file_one="bucket-file-one"
-  local bucket_file_two="bucket-file-two"
-
-  if [[ $RECREATE_BUCKETS == false ]]; then
-    run abort_all_multipart_uploads "$BUCKET_ONE_NAME"
-    assert_success
-  fi
-
-  run create_test_files "$bucket_file_one" "$bucket_file_two"
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two"
-  assert_success
-}
-
-@test "test-multipart-upload-from-bucket" {
-  local bucket_file="bucket-file"
-
-  run create_test_file "$bucket_file"
-  assert_success
-
-  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
-  assert_success
-
-  run get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy"
-  assert_success
-
-  run compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file
-  assert_success
-}
-
-@test "test_multipart_upload_from_bucket_range_too_large" {
-  local bucket_file="bucket-file"
-  run create_large_file "$bucket_file"
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file"
-  assert_success
-}
-
-@test "test_multipart_upload_from_bucket_range_valid" {
-  local bucket_file="bucket-file"
-  run create_large_file "$bucket_file"
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file"
-  assert_success
-}
-
 @test "test-presigned-url-utf8-chars" {
   test_common_presigned_url_utf8_chars "aws"
 }
@@ -324,31 +183,10 @@ export RUN_USERS=true
 #  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
 #}
 
-@test "test_head_bucket" {
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info"
-  log 5 "INFO:  $bucket_info"
-  region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region"
-  [[ $region != "" ]] || fail "empty bucket region"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-}
-
 @test "test_retention_bypass" {
   test_retention_bypass_aws_root
 }
 
-@test "test_head_bucket_doesnt_exist" {
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$?
-  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
-  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-}
-
 @test "test_add_object_metadata" {
   object_one="object-one"
   test_key="x-test-data"

tests/test_s3cmd.sh

@@ -109,10 +109,6 @@ export RUN_USERS=true
 #  test_common_presigned_url_utf8_chars "s3cmd"
 #}
 
-@test "test_list_objects_file_count" {
-  test_common_list_objects_file_count "s3cmd"
-}
-
 @test "test_get_bucket_info_s3cmd" {
   run setup_bucket "s3cmd" "$BUCKET_ONE_NAME"
   assert_success

tests/test_s3cmd_file_count.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/test_common.sh
+
+export RUN_S3CMD=true
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "s3cmd"
+}