Merge pull request #2163 from versity/test/jailing

test: jailing (all code)
2026-07-02 16:54:25 +00:00 · 2026-06-10 20:24:13 -07:00
parent 619fdb8dce 3a35e37b5c
commit fa789478d2
14 changed files with 262 additions and 139 deletions
@@ -41,7 +41,7 @@ jobs:
      - name: Install BATS
        run: |
          git clone https://github.com/bats-core/bats-core.git
-          cd bats-core && ./install.sh $HOME
+          cd bats-core && ./install.sh ${{ github.workspace }}
          git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
          git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert

@@ -69,7 +69,13 @@ jobs:
          ./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

-      - name: Build and run
+      - name: Create test user
+        run: |
+          sudo useradd -m -s /bin/bash tester
+          sudo mkdir -p /home/tester/test-files
+          sudo chown -R tester:tester /home/tester
+
+      - name: Write env file for test user
        env:
          IAM_TYPE: ${{ matrix.IAM_TYPE }}
          RUN_SET: ${{ matrix.RUN_SET }}
@@ -79,62 +85,141 @@ jobs:
          BACKEND: ${{ matrix.BACKEND }}
          RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
          DELETE_BUCKETS_AFTER_TEST: ${{ matrix.DELETE_BUCKETS_AFTER_TEST }}
-          CERT: ${{ github.workspace }}/cert.pem
-          KEY: ${{ github.workspace }}/versitygw.pem
+          CERT: /home/tester/test-files/cert.pem
+          KEY: /home/tester/test-files/versitygw.pem
          LOCAL_FOLDER: /tmp/gw
          BUCKET_ONE_NAME: versity-gwtest-bucket-one
          BUCKET_TWO_NAME: versity-gwtest-bucket-two
-          USERS_FOLDER: /tmp/iam
+          USERS_FOLDER: /home/tester/test-files/iam
          USERS_BUCKET: versity-gwtest-iam
          AWS_ENDPOINT_URL: https://127.0.0.1:7070
          PORT: 7070
          S3CMD_CONFIG: tests/s3cfg.local.default
          MC_ALIAS: versity
          LOG_LEVEL: 4
-          GOCOVERDIR: ${{ github.workspace }}/cover
+          GOCOVERDIR: /home/tester/test-files/cover
          USERNAME_ONE: HIJKLMN
          PASSWORD_ONE: 1234567
          USERNAME_TWO: OPQRSTU
          PASSWORD_TWO: 8901234
-          TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
+          TEST_FILE_FOLDER: /home/tester/test-files/versity-gwtest-files
          REMOVE_TEST_FILE_FOLDER: true
-          VERSIONING_DIR: ${{ github.workspace }}/versioning
-          COMMAND_LOG: command.log
-          TIME_LOG: time.log
-          PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
+          VERSIONING_DIR: /home/tester/test-files/versioning
+          COMMAND_LOG: /home/tester/test-files/command.log
+          TIME_LOG: /home/tester/test-files/time.log
+          PYTHON_ENV_FOLDER: /home/tester/test-files/env
          AUTOGENERATE_USERS: true
          USER_AUTOGENERATION_PREFIX: github-actions-test-
          AWS_REGION: ${{ matrix.AWS_REGION }}
-          COVERAGE_LOG: coverage.log
+          COVERAGE_LOG: /home/tester/test-files/coverage.log
          TEMPLATE_MATRIX_FILE: ${{ github.workspace }}/tests/templates/matrix.yaml
        run: |
-          make testbin
-          export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
-          export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-          export AWS_REGION=$AWS_REGION
-          export AWS_ACCESS_KEY_ID_TWO=user
-          export AWS_SECRET_ACCESS_KEY_TWO=pass
-          export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
-          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
-          aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
-          aws configure set aws_region $AWS_REGION --profile versity
-          mkdir $LOCAL_FOLDER
-          export WORKSPACE=$GITHUB_WORKSPACE
-          openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
-          openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          mkdir $GOCOVERDIR $USERS_FOLDER
-          if [[ $RECREATE_BUCKETS == "false" ]]; then
-            BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
-          fi
-          BYPASS_ENV_FILE=true $HOME/bin/bats ${{ github.workspace }}/$RUN_SET
-          if [ -e "$COVERAGE_LOG" ]; then
-            cat $COVERAGE_LOG
-          fi
+          cat > /home/tester/test-files/.env <<EOF
+          IAM_TYPE=$IAM_TYPE
+          RUN_SET=$RUN_SET
+          AWS_PROFILE=$AWS_PROFILE
+          VERSITY_EXE=$VERSITY_EXE
+          RUN_VERSITYGW=$RUN_VERSITYGW
+          BACKEND=$BACKEND
+          RECREATE_BUCKETS=$RECREATE_BUCKETS
+          DELETE_BUCKETS_AFTER_TEST=$DELETE_BUCKETS_AFTER_TEST
+          CERT=$CERT
+          KEY=$KEY
+          LOCAL_FOLDER=$LOCAL_FOLDER
+          BUCKET_ONE_NAME=$BUCKET_ONE_NAME
+          BUCKET_TWO_NAME=$BUCKET_TWO_NAME
+          USERS_FOLDER=$USERS_FOLDER
+          USERS_BUCKET=$USERS_BUCKET
+          AWS_ENDPOINT_URL=$AWS_ENDPOINT_URL
+          PORT=$PORT
+          S3CMD_CONFIG=$S3CMD_CONFIG
+          MC_ALIAS=$MC_ALIAS
+          LOG_LEVEL=$LOG_LEVEL
+          GOCOVERDIR=$GOCOVERDIR
+          USERNAME_ONE=$USERNAME_ONE
+          PASSWORD_ONE=$PASSWORD_ONE
+          USERNAME_TWO=$USERNAME_TWO
+          PASSWORD_TWO=$PASSWORD_TWO
+          TEST_FILE_FOLDER=$TEST_FILE_FOLDER
+          REMOVE_TEST_FILE_FOLDER=$REMOVE_TEST_FILE_FOLDER
+          VERSIONING_DIR=$VERSIONING_DIR
+          COMMAND_LOG=$COMMAND_LOG
+          TIME_LOG=$TIME_LOG
+          PYTHON_ENV_FOLDER=$PYTHON_ENV_FOLDER
+          AUTOGENERATE_USERS=$AUTOGENERATE_USERS
+          USER_AUTOGENERATION_PREFIX=$USER_AUTOGENERATION_PREFIX
+          AWS_REGION=$AWS_REGION
+          COVERAGE_LOG=$COVERAGE_LOG
+          TEMPLATE_MATRIX_FILE=$TEMPLATE_MATRIX_FILE
+          EOF
+          sudo chown tester:tester /home/tester/test-files/.env
+          sudo chmod 600 /home/tester/test-files/.env

+      - name: Build and run
+        run: |
+          set -euo pipefail
+          
+          make testbin
+          # github-actions normally sets this to 700, change so jailed tester can execute versitygw
+          sudo chmod 755 "$HOME"
+          
+          GO_BIN_DIR="$(dirname "$(which go)")"
+          setsid sudo -u tester -H bash -lc '
+            set -euo pipefail
+          
+            export PATH="'"$GO_BIN_DIR"':$PATH"
+            cd ${{ github.workspace }}
+            go mod download
+          
+            set -a
+            source /home/tester/test-files/.env
+            set +a
+          
+            mkdir $LOCAL_FOLDER $GOCOVERDIR $USERS_FOLDER
+          
+            export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
+            export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
+            export AWS_ACCESS_KEY_ID_TWO=user
+            export AWS_SECRET_ACCESS_KEY_TWO=pass
+            
+            aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
+            aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
+            aws configure set aws_region $AWS_REGION --profile versity
+            
+            openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
+            openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
+          
+            if [[ $RECREATE_BUCKETS == "false" ]]; then
+              BYPASS_ENV_FILE=true bash ${{ github.workspace }}/tests/setup_static.sh
+            fi
+            BYPASS_ENV_FILE=true ${{ github.workspace }}/bin/bats ${{ github.workspace }}/$RUN_SET
+          
+            if [ -e "$COVERAGE_LOG" ]; then
+              cat $COVERAGE_LOG
+            fi
+          ' &
+          pid=$!
+          (
+            sleep 1200
+            echo "=== TIMEOUT DIAGNOSTICS ==="
+            ps -ef --forest || true
+            pgrep -af bash || true
+            pgrep -af versitygw || true
+            kill -TERM -"$pid" 2>/dev/null || true
+            sleep 10
+            kill -KILL -"$pid" 2>/dev/null || true
+          ) &
+          watchdog=$!
+          
+          wait "$pid"
+          status=$?
+          kill "$watchdog" 2>/dev/null || true
+          exit "$status"
+          
      - name: Ensure coverage file exists, and generate working name
        id: prepare_coverage
        run: |
-          touch coverage.log
+          touch /home/tester/test-files/coverage.log
          run_set="${{ matrix.RUN_SET }}"
          SAFE_RUN_SET="${run_set//\//-}"
          echo "safe_run_set=$SAFE_RUN_SET" >> $GITHUB_OUTPUT
@@ -143,18 +228,18 @@ jobs:
        uses: actions/upload-artifact@v7
        with:
          name: coverage-${{ steps.prepare_coverage.outputs.safe_run_set }}-${{ matrix.RECREATE_BUCKETS }}
-          path: coverage.log
+          path: /home/tester/test-files/coverage.log
          retention-days: 1

      - name: Time report
        run: | 
-          if [ -e ${{ github.workspace }}/time.log ]; then
-            cat ${{ github.workspace }}/time.log
+          if [ -e /home/tester/test-files/time.log ]; then
+            cat /home/tester/test-files/time.log
          fi

      - name: Coverage report
        run: |
-          go tool covdata percent -i=cover
+          go tool covdata percent -i=/home/tester/test-files/cover

  print-coverage:
    runs-on: ubuntu-latest
@@ -101,7 +101,7 @@ create_bucket_object_lock_enabled() {
  fi

  local exit_code=0
-  error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$?
+  error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket 2>&1) || local exit_code=$?
  if [ $exit_code -ne 0 ]; then
    log 2 "error creating bucket: $error"
    return 1
@@ -128,9 +128,11 @@ create_bucket_rest_expect_error() {
  if ! check_param_count_v2 "bucket name, params, response code, error code, message" 5 $#; then
    return 1
  fi
+  local response
+
  env_vars="BUCKET_NAME=$1 $2"
-  if ! send_rest_command_expect_error "$env_vars" "./tests/rest_scripts/create_bucket.sh" "$3" "$4" "$5"; then
-    log 2 "error sending REST command and checking error"
+  if ! response=$(send_rest_command_expect_error "$env_vars" "./tests/rest_scripts/create_bucket.sh" "$3" "$4" "$5" 2>&1); then
+    log 2 "error sending REST command and checking error: $response"
    return 1
  fi
  return 0
@@ -140,9 +142,11 @@ create_bucket_rest_expect_success() {
  if ! check_param_count_v2 "bucket name, params" 2 $#; then
    return 1
  fi
+  local response
+
  env_vars="BUCKET_NAME=$1 $2"
-  if ! send_rest_command_expect_success "$env_vars" "./tests/rest_scripts/create_bucket.sh" "200"; then
-    log 2 "error sending REST command and checking error"
+  if ! response=$(send_rest_command_expect_success "$env_vars" "./tests/rest_scripts/create_bucket.sh" "200" 2>&1); then
+    log 2 "REST create bucket command error: $response"
    return 1
  fi
  return 0
@@ -28,18 +28,30 @@ get_object_legal_hold() {
 }

 get_object_legal_hold_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'get_object_legal_hold_rest' requires bucket, key"
+  if ! check_param_count_v2 "bucket, key" 2 $#; then
    return 1
  fi
-  if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/legal_hold.txt" ./tests/rest_scripts/get_object_legal_hold.sh); then
-    log 2 "error getting object legal hold: $result"
+  local response output_file response_code legal_hold
+
+  if ! response=$(get_file_name 2>&1); then
+    log 2 "error getting output file name: $response"
    return 1
  fi
-  if [ "$result" != "200" ]; then
-    log 2 "get-object-legal-hold returned code $result: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")"
+  output_file="$response"
+
+  if ! response=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/$output_file" ./tests/rest_scripts/get_object_legal_hold.sh); then
+    log 2 "error getting object legal hold: $response"
    return 1
  fi
+  response_code="$response"
+
+  legal_hold=$(cat "$TEST_FILE_FOLDER/$output_file")
+  if [ "$response_code" != "200" ]; then
+    log 2 "get-object-legal-hold returned code '$response_code', data: '$legal_hold'"
+    return 1
+  fi
+
+  echo "$legal_hold"
  return 0
 }

@@ -60,14 +72,26 @@ get_object_legal_hold_rest_version_id() {
  if ! check_param_count "get_object_legal_hold_rest_version_id" "bucket, key, version ID" 3 $#; then
    return 1
  fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/legal_hold.txt" ./tests/rest_scripts/get_object_legal_hold.sh); then
-    log 2 "error getting object legal hold: $result"
+  local response output_file response_code legal_hold
+
+  if ! response=$(get_file_name 2>&1); then
+    log 2 "error getting output file name: $response"
    return 1
  fi
-  legal_hold=$(cat "$TEST_FILE_FOLDER/legal_hold.txt")
-  if [ "$result" != "200" ]; then
-    log 2 "get-object-legal-hold returned code $result: $legal_hold)"
+  output_file="$response"
+
+  if ! response=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/$output_file" ./tests/rest_scripts/get_object_legal_hold.sh); then
+    log 2 "error getting object legal hold: $response"
    return 1
  fi
+  response_code="$response"
+
+  legal_hold=$(cat "$TEST_FILE_FOLDER/$output_file")
+  if [ "$response_code" != "200" ]; then
+    log 2 "get-object-legal-hold returned code '$response_code', data: '$legal_hold')"
+    return 1
+  fi
+
+  echo "$legal_hold"
  return 0
 }
@@ -166,7 +166,7 @@ setup_bucket_v3() {
    return 1
  fi
  if ! error=$(bucket_cleanup_if_bucket_exists_v2 "$1" 2>&1); then
-    log 2 "error cleaning up bucket(s), if it/they exist(s): $error"
+    log 2 "error cleaning up bucket(s), if it/they exist(s): '$error'"
    return 1
  fi
  if [ "$RECREATE_BUCKETS" == "false" ]; then
@@ -15,20 +15,15 @@
 # under the License.

 check_legal_hold_without_lock_enabled() {
-  if ! check_param_count_v2 "bucket, key, expected error" 3 $#; then
+  if ! check_param_count_v2 "bucket, key, expected code, expected error" 4 $#; then
    return 1
  fi
-  if get_object_legal_hold_rest "$1" "$2"; then
-    log 2 "get legal hold using REST succeeded without lock enabled"
-    return 1
-  fi
-  log 5 "legal hold info: $(cat "$TEST_FILE_FOLDER/legal_hold.txt")"
-  if ! code=$(xmllint --xpath '//*[local-name()="Code"]/text()' "$TEST_FILE_FOLDER/legal_hold.txt" 2>&1); then
-    log 2 "error getting error code: $code"
-    return 1
-  fi
-  if [ "$code" != "$3" ]; then
-    log 2 "code mismatch (expected '$3', actual '$code')"
+  local env_vars
+
+  env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2"
+
+  if ! send_rest_command_expect_error "$env_vars" ./tests/rest_scripts/get_object_legal_hold.sh "400" "$3" "$4"; then
+    log 2 "error sending get object legal hold command, checking error"
    return 1
  fi
  return 0
@@ -38,22 +33,20 @@ check_remove_legal_hold_versions() {
  if ! check_param_count "check_remove_legal_hold_versions" "bucket, key, version ID" 3 $#; then
    return 1
  fi
-  if ! get_object_legal_hold_rest_version_id "$1" "$2" "$3"; then
+  local response legal_hold_data
+
+  if ! response=$(get_object_legal_hold_rest_version_id "$1" "$2" "$3" 2>&1); then
    # shellcheck disable=SC2154
-    log 5 "legal hold: $legal_hold"
-    if [[ "$legal_hold" != *"MethodNotAllowed"* ]] && [[ "$legal_hold" != *"NoSuchObjectLockConfiguration"* ]]; then
-      log 2 "error getting object legal hold status with version id"
+    log 5 "legal hold: $response"
+    if [[ "$response" != *"MethodNotAllowed"* ]] && [[ "$response" != *"NoSuchObjectLockConfiguration"* ]]; then
+      log 2 "error getting object legal hold status with version id: $response"
      return 1
    fi
    return 0
  fi
-  log 5 "legal hold: $legal_hold"
-  echo -n "$legal_hold" > "$TEST_FILE_FOLDER/legal_hold.xml"
-  if ! status=$(get_element_text "$TEST_FILE_FOLDER/legal_hold.xml" "LegalHold" "Status"); then
-    log 2 "error getting XML legal hold status"
-    return 1
-  fi
-  if [ "$status" == "ON" ]; then
+  legal_hold_data="$response"
+
+  if check_xml_element_inside_string "$legal_hold_data" "ON" "LegalHold" "Status"; then
    if ! put_object_legal_hold_rest_version_id "$1" "$2" "$3" "OFF"; then
      log 2 "error removing legal hold of version ID"
      return 1
@@ -107,7 +107,7 @@ add_version_tags_check_version_id() {
    return 1
  fi
  # shellcheck disable=SC2154
-  if ! send_rest_go_command_callback "200" "check_header_version_id" "-bucketName" "$1" "-objectKey" "$2" "-debug" "-logFile" "signature.log" \
+  if ! send_rest_go_command_callback "200" "check_header_version_id" "-bucketName" "$1" "-objectKey" "$2" \
        "-method" "GET" "-query" "tagging=&versionId=$version_id" "-tagKey" "key" "-tagValue" "value" "-contentMD5" "--" "$version_id"; then
    log 2 "error tagging object"
    return 1
@@ -198,18 +198,26 @@ chunked_upload_success() {
  if ! check_param_count_v2 "data file, bucket name, key" 3 $#; then
    return 1
  fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" \
+  local response openssl_file
+
+  if ! response=$(get_file_name 2>&1); then
+    log 2 "error getting response: $response"
+    return 1
+  fi
+  openssl_file="$response"
+
+  if ! response=$(COMMAND_LOG="$COMMAND_LOG" \
         AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
         AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
         AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \
         DATA_FILE="$1" \
         BUCKET_NAME="$2" \
-         OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then
-    log 2 "error creating command: $result"
+         OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/$openssl_file" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then
+    log 2 "error creating command: $response"
    return 1
  fi

-  if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then
+  if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/$openssl_file" 200; then
    log 2 "error sending command via openssl or checking response code"
    return 1
  fi
@@ -52,12 +52,12 @@ tag_old_version() {
  fi

  version_id="$response"
-  if ! send_rest_go_command "200" "-bucketName" "$1" "-objectKey" "$2" "-query" "versionId=$version_id" "-debug" "-logFile" "signature.log" \
+  if ! send_rest_go_command "200" "-bucketName" "$1" "-objectKey" "$2" "-query" "versionId=$version_id" \
        "-commandType" "putObjectTagging" "-tagKey" "key" "-tagValue" "value" "-contentMD5"; then
    log 2 "error tagging object"
    return 1
  fi
-  if ! send_rest_go_command "200" "-bucketName" "$1" "-objectKey" "$2" "-debug" "-logFile" "signature.log" \
+  if ! send_rest_go_command "200" "-bucketName" "$1" "-objectKey" "$2" \
        "-method" "GET" "-query" "tagging=&versionId=$version_id" "-tagKey" "key" "-tagValue" "value" "-contentMD5"; then
    log 2 "error tagging object"
    return 1
@@ -58,17 +58,20 @@ send_rest_command() {
  if ! check_param_count_v2 "env vars, script" 2 $#; then
    return 1
  fi
+  local response output_file response_code
+
  if [[ "$1" == *"OUTPUT_FILE"* ]]; then
-    if ! output_file=$(echo -n "$1" | sed -n 's/^.*OUTPUT_FILE=\([^ ]*\).*$/\1/p' 2>&1); then
-      log 2 "error getting output file: $output_file"
+    if ! response=$(echo -n "$1" | sed -n 's/^.*OUTPUT_FILE=\([^ ]*\).*$/\1/p' 2>&1); then
+      log 2 "error getting output file: $response"
    fi
+    output_file="$response"
    log 5 "output file: $output_file"
  else
-    if ! file_name=$(get_file_name 2>&1); then
-      log 2 "error getting file name: $file_name"
+    if ! response=$(get_file_name 2>&1); then
+      log 2 "error getting file name: $response"
      return 1
    fi
-    output_file="$TEST_FILE_FOLDER/$file_name"
+    output_file="$TEST_FILE_FOLDER/$response"
  fi
  local env_array=("env" "COMMAND_LOG=$COMMAND_LOG" "OUTPUT_FILE=$output_file")
  if [ "$1" != "" ]; then
@@ -76,21 +79,35 @@ send_rest_command() {
    env_array+=("${env_vars[@]}")
  fi
  # shellcheck disable=SC2068
-  if ! result=$(${env_array[@]} "$2" 2>&1); then
-    log 2 "error sending command: $result"
+  if ! response=$(${env_array[@]} "$2" 2>&1); then
+    log 2 "error sending command: $response"
+    if [ -e "$output_file" ]; then
+      log 2 "output data: '$(cat "$output_file")'"
+    fi
    return 1
  fi
+  response_code="$response"
+
+  echo "$response_code"
+  echo "$output_file"
+  return 0
 }

 send_rest_command_expect_error() {
  if ! check_param_count_v2 "env vars, script, response code, error, message" 5 $#; then
    return 1
  fi
-  if ! send_rest_command "$1" "$2"; then
-    log 2 "error sending REST command"
+  local response response_lines response_code output_file
+
+  if ! response=$(send_rest_command "$1" "$2" 2>&1); then
+    log 2 "error sending REST command: $response"
    return 1
  fi
-  if ! check_rest_expected_error "$result" "$output_file" "$3" "$4" "$5"; then
+  mapfile -t response_lines <<< "$response"
+  response_code="${response_lines[0]}"
+  output_file="${response_lines[1]}"
+
+  if ! check_rest_expected_error "$response_code" "$output_file" "$3" "$4" "$5"; then
    log 2 "error checking REST error"
    return 1
  fi
@@ -124,10 +141,15 @@ send_rest_command_expect_header_error() {
  if ! check_param_count_v2 "env vars, script, response code, message" 4 $#; then
    return 1
  fi
-  if ! send_rest_command "$1" "$2"; then
-    log 2 "error sending REST command"
+  local response response_lines output_file
+
+  if ! response=$(send_rest_command "$1" "$2" 2>&1); then
+    log 2 "error sending REST command: $response"
    return 1
  fi
+  mapfile -t response_lines <<< "$response"
+  output_file="${response_lines[1]}"
+
  if ! check_rest_expected_header_error "$output_file" "$3" "$4"; then
    log 2 "error checking REST error"
    return 1
@@ -136,18 +158,24 @@ send_rest_command_expect_header_error() {
 }

 send_rest_command_expect_success() {
-if ! check_param_count_v2 "env vars, script, response code" 3 $#; then
-  return 1
-fi
-if ! send_rest_command "$1" "$2"; then
-  log 2 "error sending REST command"
-  return 1
-fi
-if [ "$result" != "$3" ]; then
-  log 2 "expected '$3', was '$result' ($(cat "$output_file"))"
-  return 1
-fi
-return 0
+  if ! check_param_count_v2 "env vars, script, response code" 3 $#; then
+    return 1
+  fi
+  local response response_lines response_code output_file
+
+  if ! response=$(send_rest_command "$1" "$2" 2>&1); then
+    log 2 "error sending REST command: $response"
+    return 1
+  fi
+  mapfile -t response_lines <<< "$response"
+  response_code="${response_lines[0]}"
+  output_file="${response_lines[1]}"
+
+  if [ "$response_code" != "$3" ]; then
+    log 2 "expected '$3', was '$response_code' ($(cat "$output_file"))"
+    return 1
+  fi
+  return 0
 }

 send_rest_command_expect_success_callback() {
@@ -171,7 +199,7 @@ send_rest_command_expect_success_callback() {
  log 5 "command: ${env_array[*]} $2"
  # shellcheck disable=SC2068
  if ! response=$(${env_array[@]} "$2" 2>&1); then
-    log 2 "error sending command: $result"
+    log 2 "error sending command: $response"
    return 1
  fi
  http_response="$response"
@@ -46,7 +46,7 @@ fi
 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature

-curl_command+=(curl -ks -w "\"%{http_code}\"")
+curl_command+=(curl -ksS -w "\"%{http_code}\"")
 url="'$AWS_ENDPOINT_URL"
 if [ "$queries" != "" ]; then
  url+="?$queries"
@@ -312,16 +312,6 @@ check_chunks_and_signatures_in_test_mode() {
  esac
 }

-record_command_lines() {
-  while IFS= read -r line; do
-    if ! mask_arg_array "$line"; then
-      return 1
-    fi
-    # shellcheck disable=SC2154
-    echo "${masked_args[*]}" >> "$COMMAND_LOG"
-  done <<< "$command"
-}
-
 build_initial_command() {
  command="PUT /$bucket_name/$key HTTP/1.1\r
 Host: $host\r
@@ -345,15 +335,6 @@ fi
  echo -en "$command" > "$COMMAND_FILE"
 }

-complete_command() {
-  echo -e "\r" >> "$COMMAND_FILE"
-  if [ -n "$COMMAND_LOG" ]; then
-    if ! record_command_lines; then
-      return 1
-    fi
-  fi
-}
-
 load_parameters

 if ! get_file_size_and_content_length; then
@@ -374,10 +355,7 @@ if ! build_chunks "$first_signature"; then
  log_rest 2 "error building chunks"
  exit 1
 fi
-if ! complete_command; then
-  log_rest 2 "error adding chunks"
-  exit 1
-fi
+echo -e "\r" >> "$COMMAND_FILE"

 if [ "$test_mode" == "true" ]; then
  log_rest 4 "TEST PASS"
@@ -83,6 +83,10 @@ setup() {
    return 1
  fi

+  if [ -n "$TEST_LOG_FILE" ]; then
+    printf "\n%s\n\n" "**** $TEST_ID ****" >> "$TEST_LOG_FILE.$TEST_ID"
+  fi
+
  if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ]; then
    if ! static_user_versitygw_setup; then
      log 2 "error setting up static versitygw users"
@@ -496,7 +496,6 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro

  run send_openssl_go_command "200" "-bucketName" "$bucket_name" "-objectKey" "$test_file" "-commandType" "putObject" \
    "-payloadFile" "$TEST_FILE_FOLDER/$test_file" "-omitPayloadTrailer" \
-    "-debug" "-logFile" "tagging.log" "-checksumType" "crc64nvme" \
-    "-payloadType" "STREAMING-UNSIGNED-PAYLOAD-TRAILER" "-chunkSize" "8192"
+    "-checksumType" "crc64nvme" "-payloadType" "STREAMING-UNSIGNED-PAYLOAD-TRAILER" "-chunkSize" "8192"
  assert_success
 }
@@ -39,7 +39,7 @@ source ./tests/drivers/create_bucket/create_bucket_rest.sh
  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
  assert_success

-  run check_legal_hold_without_lock_enabled "$bucket_name" "$test_file" "InvalidRequest"
+  run check_legal_hold_without_lock_enabled "$bucket_name" "$test_file" "InvalidRequest" "Bucket is missing Object Lock Configuration"
  assert_success
 }

@@ -62,6 +62,6 @@ source ./tests/drivers/create_bucket/create_bucket_rest.sh
  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
  assert_success

-  run check_legal_hold_without_lock_enabled "$bucket_name" "$test_file" "NoSuchObjectLockConfiguration"
+  run check_legal_hold_without_lock_enabled "$bucket_name" "$test_file" "NoSuchObjectLockConfiguration" "does not have a ObjectLock configuration"
  assert_success
 }