From a75112f88346461b44f0b3b44fd15e3babf23b67 Mon Sep 17 00:00:00 2001
From: Luke McCrone <lrm110484@gmail.com>
Date: Tue, 27 Feb 2024 18:34:19 -0300
Subject: [PATCH] test: s3cmd additional tests, include in github-actions,
 dockerfile

---
 .github/workflows/system.yml |  6 ++-
 Dockerfile_test_bats         |  8 +++-
 tests/.env.default           |  4 +-
 tests/posix_tests.sh         |  4 +-
 tests/run.sh                 | 12 ++++--
 tests/run_all.sh             |  4 ++
 tests/run_static.sh          | 12 ++++--
 tests/s3_bucket_tests.sh     | 72 ++++++----------------------------
 tests/s3cfg.local.default    |  8 ++++
 tests/s3cmd_tests.sh         | 26 +++++++++++++
 tests/setup.sh               | 15 +++++++-
 tests/setup_static.sh        |  4 +-
 tests/teardown_static.sh     |  4 +-
 tests/test_common.sh         | 75 +++++++++++++++++++++++++++++++++++-
 tests/util.sh                | 72 +++++++++++++++++++++++++---------
 tests/util_file.sh           | 38 +++++++++++++++---
 tests/util_posix.sh          |  4 +-
 17 files changed, 261 insertions(+), 107 deletions(-)
 create mode 100644 tests/s3cfg.local.default

diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml
index 18005dc1..e6500306 100644
--- a/.github/workflows/system.yml
+++ b/.github/workflows/system.yml
@@ -29,6 +29,10 @@ jobs:
           git clone https://github.com/bats-core/bats-core.git
           cd bats-core && ./install.sh $HOME
 
+      - name: Install s3cmd
+        run: |
+          sudo apt-get install s3cmd
+
       - name: Build and run
         run: |
           make testbin
@@ -42,4 +46,4 @@ jobs:
           export WORKSPACE=$GITHUB_WORKSPACE
           openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
           openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          ./tests/run_all.sh
+          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
diff --git a/Dockerfile_test_bats b/Dockerfile_test_bats
index 8a439d74..b7673e0a 100644
--- a/Dockerfile_test_bats
+++ b/Dockerfile_test_bats
@@ -1,5 +1,7 @@
 FROM --platform=linux/arm64 ubuntu:latest
 
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     git \
@@ -7,6 +9,8 @@ RUN apt-get update && \
     wget \
     curl \
     unzip \
+    tzdata \
+    s3cmd \
     jq \
     ca-certificates && \
     update-ca-certificates && \
@@ -44,7 +48,8 @@ USER tester
 COPY --chown=tester:tester . /home/tester
 
 WORKDIR /home/tester
-RUN cp tests/.env.docker tests/.env
+RUN cp tests/.env.docker.default tests/.env.docker
+RUN cp tests/s3cfg.local.default tests/s3cfg.local
 RUN make
 
 RUN . tests/.secrets && \
@@ -60,5 +65,6 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
         -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
 
 ENV WORKSPACE=.
+ENV VERSITYGW_TEST_ENV=tests/.env.docker
 
 CMD ["tests/run_all.sh"]
\ No newline at end of file
diff --git a/tests/.env.default b/tests/.env.default
index 2a761429..2470913f 100644
--- a/tests/.env.default
+++ b/tests/.env.default
@@ -5,7 +5,7 @@ BACKEND=posix
 LOCAL_FOLDER=/tmp/gw
 BUCKET_ONE_NAME=versity-gwtest-bucket-one
 BUCKET_TWO_NAME=versity-gwtest-bucket-two
-RECREATE_BUCKETS=true
+#RECREATE_BUCKETS=true
 CERT=$PWD/cert.pem
 KEY=$PWD/versitygw.pem
-S3CMD_CONFIG=./s3cfg.local
\ No newline at end of file
+S3CMD_CONFIG=./tests/s3cfg.local.default
\ No newline at end of file
diff --git a/tests/posix_tests.sh b/tests/posix_tests.sh
index 61e28873..375d1bfe 100755
--- a/tests/posix_tests.sh
+++ b/tests/posix_tests.sh
@@ -50,7 +50,7 @@ source ./tests/util_posix.sh
   fi
   setup_bucket "aws" "$bucket_name" || local created=$?
   [[ $created -eq 0 ]] || fail "Error creating bucket"
-  put_object "$test_file_folder"/"$object_name" "$bucket_name"/"$object_name"  || local result="$?"
+  put_object "aws" "$test_file_folder"/"$object_name" "$bucket_name"/"$object_name"  || local result="$?"
   [[ result -eq 0 ]] || fail "Error adding object one"
 
   chmod 000 "$LOCAL_FOLDER"/"$bucket_name"/$object_name
@@ -63,7 +63,7 @@ source ./tests/util_posix.sh
   object_is_accessible "$bucket_name" $object_name || local accessible_two=$?
   [[ $accessible_two -eq 0 ]] || fail "Object should be accessible"
 
-  delete_object "$bucket_name"/$object_name
+  delete_object "aws" "$bucket_name"/$object_name
   delete_bucket_or_contents "aws" "$bucket_name"
   delete_test_files $object_name
 }
diff --git a/tests/run.sh b/tests/run.sh
index b0384e96..a1539413 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -1,12 +1,16 @@
 #!/bin/bash
 
-export VERSITYGW_TEST_ENV=$WORKSPACE/tests/.env.default
-# shellcheck source=./.env.default
-source "$VERSITYGW_TEST_ENV"
-export AWS_PROFILE BUCKET_ONE_NAME BUCKET_TWO_NAME AWS_ENDPOINT_URL
+if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
+  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
+  exit 1
+fi
+export RECREATE_BUCKETS=true
 if ! "$HOME"/bin/bats ./tests/s3_bucket_tests.sh; then
   exit 1
 fi
 if ! "$HOME"/bin/bats ./tests/posix_tests.sh; then
   exit 1
+fi
+if ! "$HOME"/bin/bats ./tests/s3cmd_tests.sh; then
+  exit 1
 fi
\ No newline at end of file
diff --git a/tests/run_all.sh b/tests/run_all.sh
index a6d815e0..74afc3c6 100755
--- a/tests/run_all.sh
+++ b/tests/run_all.sh
@@ -1,5 +1,9 @@
 #!/bin/bash
 
+if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
+  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
+  exit 1
+fi
 if ! ./tests/run.sh; then
   exit 1
 fi
diff --git a/tests/run_static.sh b/tests/run_static.sh
index ff4e5218..c1df6aac 100755
--- a/tests/run_static.sh
+++ b/tests/run_static.sh
@@ -1,10 +1,11 @@
 #!/bin/bash
 
-export VERSITYGW_TEST_ENV=$WORKSPACE/tests/.env.static
-# shellcheck source=./.env.static
-source "$VERSITYGW_TEST_ENV"
-export AWS_PROFILE BUCKET_ONE_NAME BUCKET_TWO_NAME AWS_ENDPOINT_URL
+if [[ -z "$VERSITYGW_TEST_ENV" ]]; then
+  echo "Error:  VERSITYGW_TEST_ENV parameter must be set"
+  exit 1
+fi
 result=0
+export RECREATE_BUCKETS=false
 ./tests/setup_static.sh
 if ! "$HOME"/bin/bats ./tests/s3_bucket_tests.sh; then
   result=1
@@ -12,5 +13,8 @@ fi
 if ! "$HOME"/bin/bats ./tests/posix_tests.sh; then
   result=1
 fi
+if ! "$HOME"/bin/bats ./tests/s3cmd_tests.sh; then
+  result=1
+fi
 ./tests/teardown_static.sh
 exit $result
\ No newline at end of file
diff --git a/tests/s3_bucket_tests.sh b/tests/s3_bucket_tests.sh
index 4a80f927..a8fb37c8 100755
--- a/tests/s3_bucket_tests.sh
+++ b/tests/s3_bucket_tests.sh
@@ -12,26 +12,7 @@ source ./tests/test_common.sh
 
 # test adding and removing an object on versitygw
 @test "test_put_object" {
-  local object_name="test-object"
-
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$?
-  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
-
-  create_test_files "$object_name" || local create_result=$?
-
-  object="$BUCKET_ONE_NAME"/$object_name
-  put_object "$test_file_folder"/"$object_name" "$object" || local put_object=$?
-  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
-  object_exists "$object" || local exists_result_one=$?
-  [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
-
-  delete_object "$object" || local delete_result=$?
-  [[ $delete_result -eq 0 ]] || fail "Failed to delete object"
-  object_exists "$object" || local exists_result_two=$?
-  [[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
-
-  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$object_name"
+  test_common_put_object "aws"
 }
 
 # test listing buckets on versitygw
@@ -41,35 +22,7 @@ source ./tests/test_common.sh
 
 # test listing a bucket's objects on versitygw
 @test "test_list_objects" {
-
-  object_one="test-file-one"
-  object_two="test-file-two"
-
-  create_test_files $object_one $object_two
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result_one=$?
-  [[ result_one -eq 0 ]] || fail "Error creating bucket"
-  put_object "$test_file_folder"/$object_one "$BUCKET_ONE_NAME"/"$object_one"  || local result_two=$?
-  [[ result_two -eq 0 ]] || fail "Error adding object one"
-  put_object "$test_file_folder"/$object_two "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
-  [[ result_three -eq 0 ]] || fail "Error adding object two"
-
-  list_objects "$BUCKET_ONE_NAME"
-  local object_one_found=false
-  local object_two_found=false
-  for object in "${object_array[@]}"; do
-    if [ "$object" == $object_one ]; then
-      object_one_found=true
-    elif [ "$object" == $object_two ]; then
-      object_two_found=true
-    fi
-  done
-
-  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
-  delete_test_files $object_one $object_two
-
-  if [ $object_one_found != true ] || [ $object_two_found != true ]; then
-    fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
-  fi
+  test_common_list_objects "aws"
 }
 
 # test ability to retrieve bucket ACLs
@@ -120,9 +73,9 @@ source ./tests/test_common.sh
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result_one=$?
   [[ $result_one -eq 0 ]] || fail "Error creating bucket"
 
-  put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one"  || local result_two=$?
+  put_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one"  || local result_two=$?
   [[ $result_two -eq 0 ]] || fail "Error adding object one"
-  put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
+  put_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
   [[ $result_three -eq 0 ]] || fail "Error adding object two"
 
   error=$(aws --no-verify-ssl s3api delete-objects --bucket "$BUCKET_ONE_NAME" --delete '{
@@ -133,9 +86,9 @@ source ./tests/test_common.sh
   }') || local result=$?
   [[ $result -eq 0 ]] || fail "Error deleting objects: $error"
 
-  object_exists "$BUCKET_ONE_NAME"/"$object_one" || local exists_one=$?
+  object_exists "aws" "$BUCKET_ONE_NAME"/"$object_one" || local exists_one=$?
   [[ $exists_one -eq 1 ]] || fail "Object one not deleted"
-  object_exists "$BUCKET_ONE_NAME"/"$object_two" || local exists_two=$?
+  object_exists "aws" "$BUCKET_ONE_NAME"/"$object_two" || local exists_two=$?
   [[ $exists_two -eq 1 ]] || fail "Object two not deleted"
 
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
@@ -179,9 +132,9 @@ source ./tests/test_common.sh
   printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-  put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
+  put_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
   [[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
-  put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
+  put_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
   [[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
 
   list_objects_s3api_v1 "$BUCKET_ONE_NAME"
@@ -210,9 +163,9 @@ source ./tests/test_common.sh
   printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two"
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-  put_object "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
+  put_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$?
   [[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one"
-  put_object "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
+  put_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$?
   [[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two"
 
   list_objects_s3api_v2 "$BUCKET_ONE_NAME"
@@ -241,7 +194,7 @@ source ./tests/test_common.sh
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
   local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
-  put_object "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
+  put_object "aws" "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
   [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
 
   get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
@@ -299,7 +252,7 @@ source ./tests/test_common.sh
   abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$?
   [[ $abort_result -eq 0 ]] || fail "Abort failed"
 
-  object_exists "$BUCKET_ONE_NAME/$bucket_file" || exists=$?
+  object_exists "aws" "$BUCKET_ONE_NAME/$bucket_file" || exists=$?
   [[ $exists -eq 1 ]] || fail "Upload file exists after abort"
 
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
@@ -370,7 +323,6 @@ source ./tests/test_common.sh
 
   local key_one
   local key_two
-  echo $uploads
   key_one=$(echo "$uploads" | jq '.Uploads[0].Key')
   key_two=$(echo "$uploads" | jq '.Uploads[1].Key')
   key_one=${key_one//\"/}
diff --git a/tests/s3cfg.local.default b/tests/s3cfg.local.default
new file mode 100644
index 00000000..eab0a6b6
--- /dev/null
+++ b/tests/s3cfg.local.default
@@ -0,0 +1,8 @@
+# Setup endpoint
+host_base = 127.0.0.1:7070
+host_bucket = 127.0.0.1:7070
+bucket_location = us-east-1
+use_https = True
+
+# Enable S3 v4 signature APIs
+signature_v2 = False
diff --git a/tests/s3cmd_tests.sh b/tests/s3cmd_tests.sh
index cd6c70e3..e3dda466 100755
--- a/tests/s3cmd_tests.sh
+++ b/tests/s3cmd_tests.sh
@@ -9,7 +9,33 @@ source ./tests/util.sh
   test_common_create_delete_bucket "s3cmd"
 }
 
+# test s3cmd put object
+@test "test_put_object_s3cmd" {
+  test_common_put_object "s3cmd"
+}
+
 # test listing buckets on versitygw
 @test "test_list_buckets_s3cmd" {
   test_common_list_buckets "s3cmd"
+}
+
+@test "test_list_objects_s3cmd" {
+  test_common_list_objects "s3cmd"
+}
+
+@test "test_multipart_upload_s3cmd" {
+
+  bucket_file="largefile"
+
+  create_large_file "$bucket_file" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test file for multipart upload"
+
+  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  put_object "s3cmd" "$test_file_folder"/$bucket_file "$BUCKET_ONE_NAME/$bucket_file" || local put_result=$?
+  [[ $put_result -eq 0 ]] || fail "failed to copy file"
+
+  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
+  delete_test_files $bucket_file
 }
\ No newline at end of file
diff --git a/tests/setup.sh b/tests/setup.sh
index ee7415cd..86b73b53 100644
--- a/tests/setup.sh
+++ b/tests/setup.sh
@@ -14,7 +14,6 @@ setup() {
       echo "Warning: no .env file found in tests folder"
     fi
   else
-    echo "$VERSITYGW_TEST_ENV"
     # shellcheck source=./.env.default
     source "$VERSITYGW_TEST_ENV"
   fi
@@ -29,7 +28,19 @@ setup() {
   eval "$base_command"
 
   versitygw_pid=$!
-  export versitygw_pid AWS_PROFILE AWS_ENDPOINT_URL LOCAL_FOLDER BUCKET_ONE_NAME BUCKET_TWO_NAME S3CMD_CONFIG
+  S3CMD_OPTS=()
+  S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
+  S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
+  S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
+  export versitygw_pid \
+    AWS_PROFILE \
+    AWS_ENDPOINT_URL \
+    LOCAL_FOLDER \
+    BUCKET_ONE_NAME \
+    BUCKET_TWO_NAME \
+    S3CMD_CONFIG \
+    S3CMD_OPTS \
+    RECREATE_BUCKETS
 }
 
 # make sure required environment variables are defined properly
diff --git a/tests/setup_static.sh b/tests/setup_static.sh
index 33695e61..0a7e89c3 100755
--- a/tests/setup_static.sh
+++ b/tests/setup_static.sh
@@ -2,6 +2,6 @@
 
 source ./tests/setup.sh
 setup
-aws s3 mb s3://"$BUCKET_ONE_NAME"
-aws s3 mb s3://"$BUCKET_TWO_NAME"
+aws --no-verify-ssl s3 mb s3://"$BUCKET_ONE_NAME"
+aws --no-verify-ssl s3 mb s3://"$BUCKET_TWO_NAME"
 teardown
diff --git a/tests/teardown_static.sh b/tests/teardown_static.sh
index 052d25b8..bef43808 100755
--- a/tests/teardown_static.sh
+++ b/tests/teardown_static.sh
@@ -2,6 +2,6 @@
 
 source ./tests/setup.sh
 setup
-aws s3 rb s3://"$BUCKET_ONE_NAME"
-aws s3 rb s3://"$BUCKET_TWO_NAME"
+aws --no-verify-ssl s3 rb s3://"$BUCKET_ONE_NAME"
+aws --no-verify-ssl s3 rb s3://"$BUCKET_TWO_NAME"
 teardown
\ No newline at end of file
diff --git a/tests/test_common.sh b/tests/test_common.sh
index b9232c4a..beba2c6b 100644
--- a/tests/test_common.sh
+++ b/tests/test_common.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bats
 
+source ./tests/util_file.sh
+
 # common test for creating, deleting buckets
 # param:  "aws" or "s3cmd"
 # pass if buckets are properly listed, fail if not
@@ -22,6 +24,37 @@ test_common_create_delete_bucket() {
   [[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
 }
 
+test_common_put_object() {
+
+  if [[ $# -ne 1 ]]; then
+    fail "put object test requires command type"
+  fi
+
+  local object_name="test-object"
+
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
+  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+
+  create_test_files "$object_name" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+
+  echo "test data" > "$test_file_folder"/"$object_name"
+
+  object="$BUCKET_ONE_NAME"/$object_name
+  put_object "$1" "$test_file_folder"/"$object_name" "$object" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
+  object_exists "$1" "$object" || local exists_result_one=$?
+  [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
+
+  delete_object "$1" "$object" || local delete_result=$?
+  [[ $delete_result -eq 0 ]] || fail "Failed to delete object"
+  object_exists "$1" "$object" || local exists_result_two=$?
+  [[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_test_files "$object_name"
+}
+
 # common test for listing buckets
 # param:  "aws" or "s3cmd"
 # pass if buckets are properly listed, fail if not
@@ -58,4 +91,44 @@ test_common_list_buckets() {
 
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
   delete_bucket_or_contents "$1" "$BUCKET_TWO_NAME"
-}
\ No newline at end of file
+}
+
+test_common_list_objects() {
+
+  if [[ $# -ne 1 ]]; then
+    echo "common test function for listing objects requires command type"
+    return 1
+  fi
+
+  object_one="test-file-one"
+  object_two="test-file-two"
+
+  create_test_files $object_one $object_two
+  echo "test data" > "$test_file_folder"/"$object_one"
+  echo "test data 2" > "$test_file_folder"/"$object_two"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result_one=$?
+  [[ result_one -eq 0 ]] || fail "Error creating bucket"
+  put_object "$1" "$test_file_folder"/$object_one "$BUCKET_ONE_NAME"/"$object_one"  || local result_two=$?
+  [[ result_two -eq 0 ]] || fail "Error adding object one"
+  put_object "$1" "$test_file_folder"/$object_two "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$?
+  [[ result_three -eq 0 ]] || fail "Error adding object two"
+
+  list_objects "$1" "$BUCKET_ONE_NAME"
+  local object_one_found=false
+  local object_two_found=false
+  # shellcheck disable=SC2154
+  for object in "${object_array[@]}"; do
+    if [ "$object" == $object_one ] || [ "$object" == "s3://$BUCKET_ONE_NAME/$object_one" ]; then
+      object_one_found=true
+    elif [ "$object" == $object_two ] || [ "$object" == "s3://$BUCKET_ONE_NAME/$object_two" ]; then
+      object_two_found=true
+    fi
+  done
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_test_files $object_one $object_two
+
+  if [ $object_one_found != true ] || [ $object_two_found != true ]; then
+    fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
+  fi
+}
diff --git a/tests/util.sh b/tests/util.sh
index 8dcdcfbc..07ecdf62 100644
--- a/tests/util.sh
+++ b/tests/util.sh
@@ -56,7 +56,7 @@ delete_bucket_recursive() {
   if [[ $1 == "aws" ]]; then
     error=$(aws --no-verify-ssl s3 rb s3://"$2" --force 2>&1) || exit_code="$?"
   elif [[ $1 == "s3cmd" ]]; then
-    error=$(s3cmd --no-check-certificate rb s3://"$2" --recursive 2>&1) || exit_code="$?"
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" --recursive 2>&1) || exit_code="$?"
   else
     echo "invalid command type '$1'"
     return 1
@@ -87,7 +87,7 @@ delete_bucket_contents() {
   if [[ $1 == "aws" ]]; then
     error=$(aws --no-verify-ssl s3 rm s3://"$2" --recursive 2>&1) || exit_code="$?"
   elif [[ $1 == "s3cmd" ]]; then
-    error=$(s3cmd --no-check-certificate del s3://"$2" --recursive 2>&1) || exit_code="$?"
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate del s3://"$2" --recursive --force 2>&1) || exit_code="$?"
   else
     echo "invalid command type $1"
     return 1
@@ -113,7 +113,7 @@ bucket_exists() {
   if [[ $1 == 'aws' ]]; then
     error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?"
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd --no-check-certificate -c "$S3CMD_CONFIG" ls s3://"$2" 2>&1) || exit_code="$?"
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?"
   else
     echo "invalid command type: $1"
     return 2
@@ -194,16 +194,23 @@ setup_bucket() {
 }
 
 # check if object exists on S3 via gateway
-# param:  object path
+# param:  command, object path
 # return 0 for true, 1 for false, 2 for error
 object_exists() {
-  if [ $# -ne 1 ]; then
-    echo "object exists check missing object name"
+  if [ $# -ne 2 ]; then
+    echo "object exists check missing command, object name"
     return 2
   fi
   local exit_code=0
   local error
-  error=$(aws --no-verify-ssl s3 ls s3://"$1" 2>&1) || exit_code="$?"
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?"
+  elif [[ $1 == 's3cmd' ]]; then
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?"
+  else
+    echo "invalid command type $1"
+    return 2
+  fi
   if [ $exit_code -ne 0 ]; then
     if [[ "$error" == "" ]] || [[ $error == *"InsecureRequestWarning"* ]]; then
       return 1
@@ -211,6 +218,9 @@ object_exists() {
       echo "error checking if object exists: $error"
       return 2
     fi
+  # s3cmd returns empty when object doesn't exist, rather than error
+  elif [[ $1 == 's3cmd' ]] && [[ $error == "" ]]; then
+    return 1
   fi
   return 0
 }
@@ -219,13 +229,21 @@ object_exists() {
 # params:  source file, destination copy location
 # return 0 for success, 1 for failure
 put_object() {
-  if [ $# -ne 2 ]; then
-    echo "put object command requires source, destination"
+  if [ $# -ne 3 ]; then
+    echo "put object command requires command type, source, destination"
     return 1
   fi
   local exit_code=0
   local error
-  error=$(aws --no-verify-ssl s3 cp "$1" s3://"$2" 2>&1) || exit_code=$?
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    echo "2: $2 3: $(dirname $3)"
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$(dirname "$3")" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [ $exit_code -ne 0 ]; then
     echo "error copying object to bucket: $error"
     return 1
@@ -241,7 +259,7 @@ check_and_put_object() {
     echo "check and put object function requires source, destination"
     return 1
   fi
-  object_exists "$2" || local exists_result=$?
+  object_exists "aws" "$2" || local exists_result=$?
   if [ "$exists_result" -eq 2 ]; then
     echo "error checking if object exists"
     return 1
@@ -260,13 +278,22 @@ check_and_put_object() {
 # param:  object path, including bucket name
 # return 0 for success, 1 for failure
 delete_object() {
-  if [ $# -ne 1 ]; then
-    echo "delete object command requires object parameter"
+  if [ $# -ne 2 ]; then
+    echo "delete object command requires command type, object parameter"
     return 1
   fi
   local exit_code=0
   local error
-  error=$(aws --no-verify-ssl s3 rm s3://"$1" 2>&1) || exit_code=$?
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3 rm s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    echo "delete object s3cmd"
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm s3://"$2" 2>&1) || exit_code=$?
+    echo "$error"
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [ $exit_code -ne 0 ]; then
     echo "error deleting object: $error"
     return 1
@@ -288,7 +315,7 @@ list_buckets() {
   if [[ $1 == "aws" ]]; then
     output=$(aws --no-verify-ssl s3 ls s3:// 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
-    output=$(s3cmd --no-check-certificate -c "$S3CMD_CONFIG" ls s3://) || exit_code=$?
+    output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://) || exit_code=$?
   else
     echo "invalid format:  $1"
     return 1
@@ -312,13 +339,20 @@ list_buckets() {
 # param:  path of bucket or folder
 # export object_array (object names) on success, return 1 for failure
 list_objects() {
-  if [ $# -ne 1 ]; then
-    echo "list objects command requires bucket or folder"
+  if [ $# -ne 2 ]; then
+    echo "list objects command requires command type, and bucket or folder"
     return 1
   fi
   local exit_code=0
   local output
-  output=$(aws --no-verify-ssl s3 ls s3://"$1" 2>&1) || exit_code=$?
+  if [[ $1 == "aws" ]]; then
+    output=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [ $exit_code -ne 0 ]; then
     echo "error listing objects: $output"
     return 1
@@ -733,7 +767,7 @@ multipart_upload_from_bucket() {
   fi
 
   for ((i=0;i<$4;i++)) {
-    put_object "$3"-"$i" "$1" || put_result=$?
+    put_object "aws" "$3"-"$i" "$1" || put_result=$?
     if [[ $put_result -ne 0 ]]; then
       echo "error putting object"
       return 1
diff --git a/tests/util_file.sh b/tests/util_file.sh
index 887fed7e..0a9e84b3 100644
--- a/tests/util_file.sh
+++ b/tests/util_file.sh
@@ -10,11 +10,7 @@ create_test_files() {
   fi
   test_file_folder=.
   if [[ -z "$GITHUB_ACTIONS" ]]; then
-    test_file_folder=${TMPDIR}versity-gwtest
-    mkdir -p "$test_file_folder" || local mkdir_result=$?
-    if [[ $mkdir_result -ne 0 ]]; then
-      echo "error creating test file folder"
-    fi
+    create_test_file_folder
   fi
   for name in "$@"; do
     touch "$test_file_folder"/"$name" || local touch_result=$?
@@ -82,3 +78,35 @@ compare_files() {
   fi
   return 1
 }
+
+create_test_file_folder() {
+  test_file_folder=${TMPDIR}versity-gwtest
+  mkdir -p "$test_file_folder" || local mkdir_result=$?
+  if [[ $mkdir_result -ne 0 ]]; then
+    echo "error creating test file folder"
+  fi
+  export test_file_folder
+}
+
+# generate 16MB file
+# input: filename
+# return 0 for success, 1 for error
+create_large_file() {
+  if [[ $# -ne 1 ]]; then
+    echo "generate large file function requires filename"
+    return 1
+  fi
+
+  test_file_folder=.
+  if [[ -z "$GITHUB_ACTIONS" ]]; then
+    create_test_file_folder
+  fi
+
+  filesize=$((160*1024*1024))
+  error=$(dd if=/dev/urandom of=$test_file_folder/"$1" bs=1024 count=$((filesize/1024))) || dd_result=$?
+  if [[ $dd_result -ne 0 ]]; then
+    echo "error creating file: $error"
+    return 1
+  fi
+  return 0
+}
diff --git a/tests/util_posix.sh b/tests/util_posix.sh
index ab1ac565..9efd8343 100644
--- a/tests/util_posix.sh
+++ b/tests/util_posix.sh
@@ -8,7 +8,7 @@ object_exists_remote_and_local() {
     echo "object existence check requires single name parameter"
     return 2
   fi
-  object_exists "$1" || local exist_result=$?
+  object_exists "aws" "$1" || local exist_result=$?
   if [[ $exist_result -eq 2 ]]; then
     echo "Error checking if object exists"
     return 2
@@ -32,7 +32,7 @@ object_not_exists_remote_and_local() {
     echo "object non-existence check requires single name parameter"
     return 2
   fi
-  object_exists "$1" || local exist_result=$?
+  object_exists "aws" "$1" || local exist_result=$?
   if [[ $exist_result -eq 2 ]]; then
     echo "Error checking if object doesn't exist"
     return 2