Merge pull request #2279 from versity/test/parallel_runs

test: running tests locally in parallel, misc fixes/changes

tests/Dockerfile_test_bats

@@ -54,6 +54,7 @@ ENV HOME=/home/tester
 RUN git clone https://github.com/bats-core/bats-core.git && \
     cd bats-core && \
     ./install.sh /home/tester
+ENV PATH="/home/tester/bin:${PATH}"
 
 # Create a shared venv & install Python deps there
 ENV VENV=/opt/venv
@@ -72,6 +73,7 @@ RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/test
 RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
 
 WORKDIR /home/tester
+RUN go mod download
 RUN make
 
 RUN . $SECRETS_FILE && \

tests/README.md

@@ -46,7 +46,7 @@
     openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
 ```
 10. The program uses two environment values for the bucket names (or prefixes):  `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME`.  Originally, and for static and some older tests, these are the full bucket names.  In later, non-static tests, these are prefixes, and bucket suffixes are auto-generated.  Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets, for older and static bucket tests, or prefixes, for newer and non-static bucket tests.  If you want static buckets, i.e. you don't want them to be re-created each test, set `RECREATE_BUCKETS` to `false`.
-11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests (not currently recommended due to long running time), run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
+11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, technically, the user can run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`, but this is not recommended due to long running time.  Instead, see the docker section below to run tests in parallel.
 12. BATS tests can also be run directly with the format `VERSIYTGW_TEST_ENV=<env file> tests/<test file name>`, or for single tests, `VERSIYTGW_TEST_ENV=<env file> tests/<file name> -f <test name>`.  Example:  `VERSITYGW_TEST_ENV=tests/.env tests/test_rest_bucket.sh -f "REST - HeadBucket"`.
 
 #### Tags
@@ -88,6 +88,10 @@ To communicate directly with s3, in order to compare the gateway results to dire
 6.  To use tag functionality, the `--tags` parameter can be passed to the container.
 7.  To troubleshoot the Docker container, use `docker run -it --entrypoint /bin/bash <image tag or ID>` to use the shell and examine the container.
 
+### In Parallel
+
+The script `run_parallel.sh` is provided to allow users to more quickly run tests in parallel on the local machine.  To use this script, first build and tag a docker container with the desired configuration.  Next, run the script with this tag, the desired tests (can find with `./tests/run.sh -l`) separated by comma, the maximum number of parallel jobs (default 4), and the folder to place the logs.
+
 ## Instructions - Running with docker-compose
 
 A file named `docker-compose-bats.yml` is provided in the root folder.  A few configurations are provided, and you can also create your own provided you have a secrets and config file:
@@ -216,6 +220,16 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 
 **BYPASS_ENV_FILE**:  skip loading `.env` file on startup, default is **false**
 
+**DIRECT_CLOUDFRONT_TAG**:  if communicating directly with S3, name of tag used for HTTPS cloudfront distributions for bucket websites
+
+**GO_COMMAND_GENERATOR_EXECUTABLE**:  location to set executable for go REST command generation.  If not used, program will run `go run ...` each time rather than compiling.  However, if using, user must ensure that executable is removed and recompiled on go code updates.
+
+**QUICK_COMPARE_SIZE**:  for some comparisons between local and endpoint files to verify correct transfer, use checksum rather than full download to compare if file or part is above this size.  If not used, then just download and compare.
+
+**WEBSITE**:  website port, if using versitygw website functionality
+
+**WEBSITE_ENDPOINT**:  website endpoint, if using versitygw website functionality
+
 ## REST Scripts
 
 REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder.  To call a script, the following parameters are needed:

tests/commands/get_bucket_policy.sh

@@ -224,13 +224,16 @@ search_for_first_policy_line_or_full_policy() {
 }
 
 get_bucket_policy_mc() {
-  if ! check_param_count "get_bucket_policy_mc" "bucket" 1 $#; then
+  if ! check_param_count_v2 "bucket" 1 $#; then
     return 1
   fi
-  bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1" 2>&1) || get_result=$?
-  if [[ $get_result -ne 0 ]]; then
-    log 2 "error getting policy: $bucket_policy"
+  local bucket="$1"
+  local response
+
+  if ! response=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$bucket" 2>&1); then
+    log 2 "error getting policy: $response"
     return 1
   fi
+  printf '%s\n' "$response"
   return 0
 }

tests/drivers/cloudfront.sh

@@ -276,7 +276,7 @@ delete_cloudfront_distribution() {
   if [ -z "$distribution_id" ] || [ "$distribution_id" == "null" ]; then
     return 0
   fi
-  if ! response="$(env AWS_IGNORE_CONFIGURED_ENDPOINT_URLS=true aws cloudfront get-distribution-config --id "$distribution_id" 2>&1)"; then
+  if ! response="$(send_command env AWS_IGNORE_CONFIGURED_ENDPOINT_URLS=true aws cloudfront get-distribution-config --id "$distribution_id" 2>&1)"; then
     if  [[ "$response" == *"The specified distribution does not exist"* ]]; then
       return 0
     fi

tests/drivers/list_objects/list_objects.sh

@@ -18,30 +18,36 @@ list_check_objects_common() {
   if ! check_param_count "list_check_objects_common" "client, bucket, object one, object two" 4 $#; then
     return 1
   fi
-  if ! response=$(list_objects "$1" "$2" 2>&1); then
+  local client="$1" bucket="$2" object_one="$3" object_two="$4"
+  local response object_list object_data
+
+  if ! response=$(list_objects "$client" "$bucket" 2>&1); then
     log 2 "error listing objects: $response"
     return 1
   fi
   log 5 "response: $response"
   mapfile -t object_list <<< "$response"
+  log 5 "object list size: ${#object_list[@]}"
 
   local object_one_found=false
   local object_two_found=false
   for object_data in "${object_list[@]}"; do
-    if [ "$1" == "s3cmd" ]; then
-      object="$(echo -n "$object_data" | awk '{ for (i=4; i<=NF; i++) printf "%s%s", $i, (i<NF ? OFS : ORS) }')"
+    if [ "$client" == "s3cmd" ]; then
+      object="$(awk '{ for (i=4; i<=NF; i++) printf "%s%s", $i, (i<NF ? OFS : ORS) }' <<< "$object_data")"
+    elif [ "$client" == "mc" ]; then
+      object="$(awk '{print $6}' <<< "$object_data")"
     else
       object="$object_data"
     fi
-    if [ "$object" == "$3" ] || [ "$object" == "s3://$2/$3" ]; then
+    if [ "$object" == "$object_one" ] || [ "$object" == "s3://$bucket/$object_one" ]; then
       object_one_found=true
-    elif [ "$object" == "$4" ] || [ "$object" == "s3://$2/$4" ]; then
+    elif [ "$object" == "$object_two" ] || [ "$object" == "s3://$bucket/$object_two" ]; then
       object_two_found=true
     fi
   done
 
   if [ $object_one_found != true ] || [ $object_two_found != true ]; then
-    log 2 "$3 and/or $4 not listed (all objects: ${object_list[*]})"
+    log 2 "'$object_one' and/or '$object_two' not listed (all objects: ${object_list[*]})"
     return 1
   fi
   return 0

tests/env.sh

@@ -36,8 +36,41 @@ base_setup() {
   return 0
 }
 
+get_log_name_from_placeholders() {
+  local test_log_file_string="$TEST_LOG_FILE_PATTERN" timestamp test_filename filename_root
+
+  if [[ "$test_log_file_string" == *"{timestamp}"* ]]; then
+    timestamp="$(date '+%Y%m%dT%H%M%S')"
+    test_log_file_string="${test_log_file_string//\{timestamp\}/$timestamp}"
+  fi
+  if [[ "$test_log_file_string" == *"{filename}"* ]]; then
+    test_filename="$(basename "$BATS_TEST_FILENAME")"
+    filename_root="${test_filename%.*}"
+    test_log_file_string="${test_log_file_string//\{filename\}/$filename_root}"
+  fi
+  export TEST_LOG_FILE="$test_log_file_string"
+}
+
+get_log_name() {
+  if [ -n "$TEST_LOG_FILE_PATTERN" ]; then
+    if [ -n "$TEST_LOG_FILE" ]; then
+      echo "both TEST_LOG_FILE_PATTERN and TEST_LOG_FILE cannot be defined"
+      return 1
+    fi
+    if [[ ( "$TEST_LOG_FILE_PATTERN" == *'{'* ) || ( "$TEST_LOG_FILE_PATTERN" == *'}'* ) ]]; then
+      get_log_name_from_placeholders
+    else
+      export TEST_LOG_FILE="$TEST_LOG_FILE_PATTERN"
+    fi
+  fi
+}
+
 setup_test_log_file() {
-  if [ -n "$TEST_LOG_FILE" ]; then
+  if [ -n "$TEST_LOG_FILE" ] || [ -n "$TEST_LOG_FILE_PATTERN" ]; then
+    if ! get_log_name; then
+      log 1 "error getting log file name"
+      return 1
+    fi
     if ! error=$(touch "$TEST_LOG_FILE.$TEST_ID" 2>&1); then
       log 1 "error creating test log file: $error"
       return 1

tests/run.sh

@@ -19,14 +19,37 @@ source ./tests/drivers/params.sh
 show_help() {
   echo "Usage: $0 [option...]"
   echo "   -h, --help                             Display this help message and exit"
-  echo "                                          Separate the below by comma"
-  echo "all                                       Attempt to run all tests (not recommended)"
+  echo "   -l                                     Only list suites (don't run)"
+  echo "   env BATS=<path>                        Use a specific bats executable path"
+  echo "   env VERSITYGW_TEST_ENV=<path>          env file to pass to tests"
+  echo "all                                       Attempt to run all tests (not recommended here due to time, use run_parallel.sh to run in parallel)"
   echo "{suite} {pattern}                         Attempt to run tests matching pattern in single suite"
+  echo "                                          Suites below (to run multiple, separate by comma)"
   echo_help_lines
 }
 
+get_bats_executable() {
+  if [ -n "$BATS" ]; then
+    if [ ! -x "$BATS" ]; then
+      echo "BATS executable '$BATS' not found or not executable" >&2
+      return 1
+    fi
+    printf '%s\n' "$BATS"
+    return 0
+  fi
+  if bats_cmd="$(command -v bats 2>/dev/null)" && [[ -x "$bats_cmd" ]]; then
+    printf '%s\n' "$bats_cmd"
+    return 0
+  fi
+  echo "unable to find bats executable; set BATS=<path>" >&2
+  return 1
+}
+
 echo_help_lines() {
-  gather_test_files
+  local run_sets_list run_sets=() run_set description spaces_needed
+
+  run_sets_list=$(get_run_sets_and_files_if_needed)
+  read -r -a run_sets <<< "$run_sets_list"
 
   for run_set in "${run_sets[@]}"; do
     description=${run_set/-/ }
@@ -38,46 +61,71 @@ echo_help_lines() {
   done
 }
 
-gather_test_files() {
+get_run_sets_and_files_if_needed() {
+  if ! check_param_count_le "'true', if files desired" 1 $#; then
+    return 1
+  fi
+  local want_files="$1"
+
+  if [ -n "$want_files" ] && [ "$want_files" != "true" ]; then
+    echo "param for printing files as well must be 'true' or unset"
+    return 1
+  fi
+
+  local f files=() file_without_header file_without_sh run_set run_sets=()
+
   while IFS= read -r f; do
     if grep -q '@test' "$f"; then
-      files+=("$f")
+      if [ "$want_files" == "true" ]; then
+        files+=("$f")
+      fi
       file_without_header=${f/tests\/test_/}
       file_without_sh=${file_without_header/.sh/}
       run_set=${file_without_sh//_/-}
       run_sets+=("$run_set")
     fi
   done < <(find tests -name 'test_*.sh' | sort)
+  printf '%s\n' "${run_sets[*]}"
+  if [ "$want_files" == "true" ]; then
+    printf '%s\n' "${files[*]}"
+  fi
 }
 
+# return 0 for complete, 1 for continue checking suites, 2 for error
 run_set_if_matching() {
-  if ! check_param_count_gt "set name, test (optional)" 2 $#; then
-    exit 1
+  if ! check_param_count_gt "desired run set, current run set name, current file name, test pattern (optional)" 3 $#; then
+    return 2
   fi
-  if [ "$1" == "all" ]; then
-    echo "running '${run_sets[$idx]}' test suite ..."
-    if ! "$HOME"/bin/bats "${files[$idx]}"; then
-      echo "error running '${files[$idx]}' tests"
-      exit 1
+  local desired_run_set="$1" current_run_set="$2" current_file="$3" test_pattern="$4" bats_executable
+
+  if ! bats_executable=$(get_bats_executable); then
+    return 2
+  fi
+
+  if [ "$desired_run_set" == "all" ]; then
+    echo "running '$current_run_set' test suite ..."
+    if ! "$bats_executable" "$current_file"; then
+      echo "error running '$current_run_set' tests" >&2
+      return 2
     fi
-    suite_run="true"
-  elif [ "$run_set" == "$1" ]; then
-    if [ "$2" != "" ]; then
-      echo "running test(s) matching '$2' in '${run_sets[$idx]}' test suite ..."
-      if ! "$HOME"/bin/bats "${files[$idx]}" "-f" "$2"; then
-        echo "error running '$2' test in '${files[$idx]}' suite"
-        exit 1
+    return 1
+  elif [ "$current_run_set" == "$desired_run_set" ]; then
+    if [ "$test_pattern" != "" ]; then
+      echo "running test(s) matching '$test_pattern' in '$current_run_set' test suite ..."
+      if ! "$bats_executable" "$current_file" "-f" "$test_pattern"; then
+        echo "error running '$test_pattern' test(s) in '$current_run_set' suite" >&2
+        return 2
       fi
     else
-      echo "running '${run_sets[$idx]}' test suite ..."
-      if ! "$HOME"/bin/bats "${files[$idx]}"; then
-        echo "error running '${files[$idx]}' suite"
-        exit 1
+      echo "running '$current_run_set' test suite ..."
+      if ! "$bats_executable" "$current_file"; then
+        echo "error running '$current_run_set' suite" >&2
+        return 2
       fi
     fi
-    complete="true"
-    suite_run="true"
+    return 0
   fi
+  return 1
 }
 
 handle_tags() {
@@ -90,27 +138,50 @@ handle_tags() {
 }
 
 handle_param() {
-  if ! check_param_count_gt "run sets, separated by comma, or single run set then test name" 1 $#; then
-    exit 1
+  if ! check_param_count_ge_le "run set, test pattern (optional)" 1 2 $#; then
+    return 1
   fi
+  local run_set="$1" test_pattern="$2"
+  local run_sets_and_files lines run_sets files idx run_result
 
-  gather_test_files
+  run_sets_and_files=$(get_run_sets_and_files_if_needed "true")
+  mapfile -t lines <<< "$run_sets_and_files"
+  read -r -a run_sets <<< "${lines[0]}"
+  read -r -a files <<< "${lines[1]}"
 
-  idx=0
-  complete="false"
-  suite_run="false"
-  for run_set in "${run_sets[@]}"; do
-    run_set_if_matching "$1" "$2"
-    if [ "$complete" == "true" ]; then
+  for idx in "${!run_sets[@]}"; do
+    run_result=0
+    run_set_if_matching "$run_set" "${run_sets[$idx]}" "${files[$idx]}" "$test_pattern" || run_result=$?
+    if [ "$run_result" -eq 0 ]; then
       break
+    elif [ "$run_result" -eq 2 ]; then
+      echo "error running set '$run_set" >&2
+      return 1
     fi
-    ((idx++))
   done
 
-  if [ "$suite_run" == "false" ]; then
-    echo "no suites matching '$1'"
-    exit 1
+  if [ "$run_result" -eq 1 ] && [ "$run_set" != "all" ]; then
+    echo "no suites matching '$run_set'" > /dev/stderr
+    return 1
   fi
+  return 0
+}
+
+list_all_test_suites() {
+  local run_set_list run_sets set_length matching_test_string=""
+
+  run_set_list=$(get_run_sets_and_files_if_needed)
+  read -r -a run_sets <<< "$run_set_list"
+  set_length=${#run_sets[@]}
+
+  if [ "$set_length" -le 0 ]; then
+    printf '\n'
+    return 0
+  fi
+
+  matching_test_string=$(IFS=","; echo "${run_sets[*]}")
+  printf '%s\n' "$matching_test_string"
+  return 0
 }
 
 if [ $# -le 0 ] || [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
@@ -125,13 +196,20 @@ if [ "$1" == "--tags" ]; then
   exit 0
 fi
 
+if [ "$1" == "-l" ]; then
+  list_all_test_suites
+  exit 0
+fi
+
 IFS=',' read -ra options <<< "$1"
 if [ "$2" != "" ] && [ "${#options[@]}" -gt 1 ]; then
-  echo "cannot call multiple suites with test name"
+  echo "cannot call multiple suites with test name" >&2
   exit 1
 fi
 for option in "${options[@]}"; do
-  handle_param "$option" "$2"
+  if ! handle_param "$option" "$2"; then
+    exit 1
+  fi
 done
 
 # shellcheck disable=SC2086

tests/run_parallel.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/drivers/params.sh
+
+DEFAULT_MAX_PARALLEL_JOBS=4
+DEFAULT_DOCKER_LOG_FOLDER="$PWD/runtime/log"
+
+check_for_finished_processes() {
+  if ! check_param_count_v2 "pid array ref, suite array ref, time array ref" 3 $#; then
+    return 1
+  fi
+  local -n pids_ref="$1" suites_ref="$2" times_ref="$3"
+  local -a pid_snapshot
+  local pid status run_end_time
+
+  pid_snapshot=("${!pids_ref[@]}")
+  for pid in "${pid_snapshot[@]}"; do
+    if ! kill -0 "$pid" 2>/dev/null; then
+      wait "$pid"
+      status=$?
+      run_end_time=$(date +%s)
+      printf '%s\n' "'$pid' (${suites_ref[$pid]}) finished with status '$status' (duration: $((run_end_time-${times_ref[$pid]}))s)"
+      unset "pids_ref[$pid]"
+      unset "suites_ref[$pid]"
+      unset "times_ref[$pid]"
+    fi
+  done
+}
+
+run_tests() {
+  if ! check_param_count_v2 "image tag, test list, max parallel jobs, docker log folder" 4 $#; then
+    return 1
+  fi
+  local image_tag="$1" test_list="$2" max_parallel_jobs="$3" docker_log_folder="$4"
+  local test_array pids suites times timestamp test_suite end_time duration
+
+  IFS=, read -r -a test_array <<< "$test_list"
+
+  declare -A pids suites times
+  timestamp="$(date '+%Y%m%dT%H%M%S')"
+
+  for test_suite in "${test_array[@]}"; do
+    while [ ${#pids[@]} -ge "$max_parallel_jobs" ]; do
+      sleep 1
+      check_for_finished_processes "pids" "suites" "times"
+    done
+    docker run -v "$PWD/runtime/config:/home/tester/config" -v "$docker_log_folder:/home/tester/log" -t "$image_tag" "$test_suite" > "$docker_log_folder/${test_suite}-${timestamp}.log" &
+    pid=$!
+    printf '%s\n' "'$test_suite' started (pid: '$pid')"
+    pids[$pid]=$pid
+    # shellcheck disable=SC2034
+    suites[$pid]=$test_suite
+    # shellcheck disable=SC2034
+    times[$pid]=$(date +%s)
+  done
+
+  while [ ${#pids[@]} -gt 0 ]; do
+    check_for_finished_processes "pids" "suites" "times"
+    sleep 1
+  done
+}
+
+if ! check_param_count_ge_le "docker image tag, test suites (separated by comma), max parallel jobs at once (default '$DEFAULT_MAX_PARALLEL_JOBS', log folder (default '$DEFAULT_DOCKER_LOG_FOLDER')" 2 4 $#; then
+  exit 1
+fi
+
+start_time=$(date +%s)
+
+image_tag="$1"
+test_list="$2"
+if [ "$3" == "" ]; then
+  max_parallel_jobs="$DEFAULT_MAX_PARALLEL_JOBS"
+else
+  if [ "$3" -le 0 ]; then
+    echo "max parallel jobs must be at least 1"
+    exit 1
+  fi
+  max_parallel_jobs="$3"
+fi
+if [ "$4" == "" ]; then
+  docker_log_folder="$DEFAULT_DOCKER_LOG_FOLDER"
+else
+  docker_log_folder="$4"
+fi
+
+if ! docker image inspect "$image_tag" >/dev/null 2>&1; then
+  echo "image tagged '$image_tag' doesn't exist"
+  exit 1
+fi
+
+if ! mkdir -p "$docker_log_folder"; then
+  echo "unable to create docker log folder '$docker_log_folder'"
+  exit 1
+fi
+
+run_tests "$image_tag" "$test_list" "$max_parallel_jobs" "$docker_log_folder"
+
+end_time=$(date +%s)
+duration=$((end_time-start_time))
+printf '%s\n' "duration: ${duration}s"

tests/setup.sh

@@ -83,11 +83,17 @@ setup() {
     return 1
   fi
 
-  if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ]; then
-    if ! static_user_versitygw_setup; then
-      log 2 "error setting up static versitygw users"
+  if [ "$RUN_USERS" == "true" ] && [ "$SKIP_USERS_TESTS" != "true" ]; then
+    if ! static_user_v1_cleanup; then
+      log 2 "error cleaning up v1 static users"
       return 1
     fi
+    if [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ] && [ "$AUTOGENERATE_USERS" == "false" ]; then
+      if ! static_user_versitygw_setup; then
+        log 2 "error setting up static versitygw users"
+        return 1
+      fi
+    fi
   fi
 
   log 4 "Running test $BATS_TEST_NAME"
@@ -136,21 +142,31 @@ teardown() {
   teardown_logs
 }
 
+static_user_v1_cleanup() {
+  if [ -n "$USERNAME_ONE" ]; then
+    if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then
+      log 2 "error deleting user '$USERNAME_ONE'"
+      return 1
+    fi
+  fi
+  if [ -n "$USERNAME_TWO" ]; then
+    if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then
+      log 2 "error deleting user '$USERNAME_TWO'"
+      return 1
+    fi
+  fi
+  return 0
+}
+
 bucket_and_user_cleanup() {
   log 4 "********** BEGIN TEARDOWN **********"
   if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then
     log 3 "error cleaning up buckets after test"
   fi
   if [ "$SKIP_USERS_TESTS" != "true" ]; then
-    if [ -n "$USERNAME_ONE" ]; then
-      if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then
-        log 3 "error deleting user '$USERNAME_ONE'"
-      fi
-    fi
-    if [ -n "$USERNAME_TWO" ]; then
-      if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then
-        log 3 "error deleting user '$USERNAME_TWO'"
-      fi
+    if ! static_user_v1_cleanup; then
+      log 3 "error cleaning up v1 static users"
+      return 1
     fi
   fi
   if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then

tests/test_command.sh

@@ -21,6 +21,7 @@ source ./tests/commands/command.sh
 source ./tests/logger.sh
 source ./tests/setup_unit.sh
 
+# tags: unit
 @test "check_for_and_or_build_go_executable" {
   export GO_COMMAND_GENERATOR_EXECUTABLE=
   run check_for_and_or_build_go_executable

tests/test_mc.sh

@@ -28,6 +28,9 @@ source ./tests/drivers/get_bucket_location/get_bucket_location.sh
 source ./tests/drivers/get_bucket_tagging/get_bucket_tagging.sh
 source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh
 source ./tests/drivers/get_object_tagging/get_object_tagging.sh
+source ./tests/drivers/head_bucket/head_bucket_mc.sh
+source ./tests/drivers/list_buckets/list_buckets.sh
+source ./tests/drivers/list_objects/list_objects.sh
 source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
 
 export RUN_MC=true

tests/test_mc_file_count.sh

@@ -17,6 +17,7 @@
 load ./bats-support/load
 load ./bats-assert/load
 
+source ./tests/drivers/list_objects/list_objects.sh
 source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
 source ./tests/test_common.sh
 

tests/test_rest_delete_objects.sh

@@ -160,6 +160,7 @@ source ./tests/setup.sh
   assert_success
 }
 
+# tags: curl, DeleteObjects
 @test "REST - DeleteObjects - quiet mode" {
   if [ "$DIRECT" != "true" ]; then
     skip "https://github.com/versity/versitygw/issues/2124"