test: logging, convert more setup commands to REST

2025-12-23 05:05:16 +00:00 · 2025-05-09 18:13:18 -03:00
parent f912778617
commit 282e875d9f
13 changed files with 164 additions and 66 deletions
--- a/tests/commands/delete_object.sh
+++ b/tests/commands/delete_object.sh
@@ -65,6 +65,23 @@ delete_object_bypass_retention() {
  return 0
 }

+# shellcheck disable=SC2317
+delete_object_rest() {
+  if ! check_param_count "delete_object_rest" "bucket, key" 2 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
+    log 2 "error deleting object: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt")
+    log 2 "expected '204', was '$result' ($delete_object_error)"
+    return 1
+  fi
+  return 0
+}
+
 delete_object_version() {
  if ! check_param_count "delete_object_version" "bucket, key, version ID" 3 $#; then
    return 1
@@ -76,6 +93,22 @@ delete_object_version() {
  return 0
 }

+delete_object_version_rest() {
+  if ! check_param_count "delete_object_version_rest" "bucket name, object name, version ID" 3 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
+    log 2 "error deleting object: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt")
+    log 2 "expected '204', was '$result' ($delete_object_error)"
+    return 1
+  fi
+  return 0
+}
+
 delete_object_version_bypass_retention() {
  if ! check_param_count "delete_object_version_bypass_retention" "bucket, key, version ID" 3 $#; then
    return 1
@@ -87,6 +120,22 @@ delete_object_version_bypass_retention() {
  return 0
 }

+delete_object_version_rest_bypass_retention() {
+  if ! check_param_count "delete_object_version_rest_bypass_retention" "bucket, key, version ID" 3 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" BYPASS_GOVERNANCE_RETENTION="true" \
+      OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
+    log 2 "error deleting object: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    log 2 "expected '204', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+    return 1
+  fi
+  return 0
+}
+
 delete_object_with_user() {
  record_command "delete-object" "client:$1"
  if ! check_param_count "delete_object_version_bypass_retention" "command type, bucket, key, access ID, secret key" 5 $#; then
@@ -124,4 +173,4 @@ delete_object_rest() {
    return 1
  fi
  return 0
-}
+}
--- a/tests/commands/put_object_legal_hold.sh
+++ b/tests/commands/put_object_legal_hold.sh
@@ -60,3 +60,18 @@ put_object_legal_hold_version_id() {
  fi
  return 0
 }
+
+put_object_legal_hold_rest_version_id() {
+  if ! check_param_count "put_object_legal_hold_rest" "bucket, key, version ID, hold status" 4 $#; then
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" STATUS="$4" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_legal_hold.sh 2>&1); then
+    log 2 "error putting object legal hold: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+    return 1
+  fi
+  return 0
+}
--- a/tests/drivers/drivers.sh
+++ b/tests/drivers/drivers.sh
@@ -20,7 +20,19 @@ check_param_count() {
    return 1
  fi
  if [ "$3" -ne "$4" ]; then
-    log 2 "function $1 requires $2"
+    log_with_stack_ref 2 "function $1 requires $2" 2
+    return 1
+  fi
+  return 0
+}
+
+assert_param_count() {
+  if [ $# -ne 4 ]; then
+    log 2 "'assert_param_count' requires function name, params list, expected, actual"
+    return 1
+  fi
+  if [ "$3" -ne "$4" ]; then
+    log_with_stack_ref 2 "function $1 requires $2" 4
    return 1
  fi
  return 0
@@ -32,7 +44,7 @@ check_param_count_gt() {
    return 1
  fi
  if [ "$3" -gt "$4" ]; then
-    log 2 "function $1 requires $2"
+    log_with_stack_ref 2 "function $1 requires $2" 2
    return 1
  fi
  return 0
--- a/tests/logger.sh
+++ b/tests/logger.sh
@@ -16,9 +16,31 @@

 # levels:  1 - crit, 2 - err, 3 - warn, 4 - info, 5 - debug, 6 - trace

+check_log_params() {
+  if [ $# -ne 4 ]; then
+    echo "'check_log_params' requires function name, params list, expected, actual"
+    return 1
+  fi
+  if [ "$3" -ne "$4" ]; then
+    echo "function $1 requires $2" 2
+    return 1
+  fi
+  return 0
+}
+
 log() {
-  if [[ $# -ne 2 ]]; then
-    echo "log function requires level, message"
+  if ! check_log_params "log" "level, message" 2 $#; then
+    return 1
+  fi
+  if ! log_with_stack_ref "$1" "$2" 2; then
+    echo "error logging with stack ref"
+    return 1
+  fi
+  return 0
+}
+
+log_with_stack_ref() {
+  if ! check_log_params "log_with_stack_ref" "level, message, stack reference" 3 $#; then
    return 1
  fi
  # shellcheck disable=SC2153
@@ -36,15 +58,14 @@ log() {
    *) echo "invalid log level $1"; return 1
  esac
  if [[ ( "$2" == *"access"* ) || ( "$2" == *"secret"* ) || ( "$2" == *"Credential="* ) ]]; then
-    log_mask "$log_level" "$2"
+    log_mask "$log_level" "$2" "$3"
    return 0
  fi
-  log_message "$log_level" "$2"
+  log_message "$log_level" "$2" "$3"
 }

 log_mask() {
-  if [[ $# -ne 2 ]]; then
-    echo "mask and log requires level, string"
+  if ! check_log_params "log_mask" "level, string, stack reference" 3 $#; then
    return 1
  fi

@@ -53,12 +74,11 @@ log_mask() {
    return 1
  fi

-  log_message "$log_level" "$masked_data"
+  log_message "$log_level" "$masked_data" "$3"
 }

 mask_args() {
-  if [ $# -ne 1 ]; then
-    echo "'mask_args' requires string"
+  if ! check_log_params "mask_args" "string" 1 $#; then
    return 1
  fi
  unmasked_array=()
@@ -101,8 +121,7 @@ mask_arg_array() {
 }

 check_arg_for_mask() {
-  if [ $# -ne 1 ]; then
-    echo "'check_arg_for_mask' requires arg"
+  if ! check_log_params "check_arg_for_mask" "arg" 1 $#; then
    return 1
  fi
  if [[ $mask_next == true ]]; then
@@ -137,16 +156,16 @@ check_arg_for_mask() {
 }

 log_message() {
-  if [[ $# -ne 2 ]]; then
-    echo "log message requires level, message"
+  if ! check_log_params "log_message" "level, message, stack reference" 3 $#; then
    return 1
  fi
+  local bash_source_ref=$(($3+1))
  now="$(date "+%Y-%m-%d %H:%M:%S")"
  if [[ ( "$1" == "CRIT" ) || ( "$1" == "ERROR" ) ]]; then
    echo "$now $1 $2" >&2
  fi
  if [[ -n "$TEST_LOG_FILE" ]]; then
-    echo "$now ${BASH_SOURCE[2]}:${BASH_LINENO[1]} $1 $2" >> "$TEST_LOG_FILE.tmp"
+    echo "$now ${BASH_SOURCE[$bash_source_ref]}:${BASH_LINENO[$3]} $1 $2" >> "$TEST_LOG_FILE.tmp"
  fi
  sync
 }
--- a/tests/rest_scripts/delete_object.sh
+++ b/tests/rest_scripts/delete_object.sh
@@ -24,11 +24,18 @@ bucket_name="$BUCKET_NAME"
 key=$(echo -n "$OBJECT_KEY" | jq -sRr 'split("/") | map(@uri) | join("/")')
 # shellcheck disable=SC2154
 bypass_governance_retention=${BYPASS_GOVERNANCE_RETENTION:=false}
+# shellcheck disable=SC2153,SC2154
+version_id="$VERSION_ID"

 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")

 #x-amz-object-attributes:ETag
-canonical_request_data+=("DELETE" "/$bucket_name/$key" "" "host:$host")
+canonical_request_data=("DELETE" "/$bucket_name/$key")
+queries=""
+if [ "$version_id" != "" ]; then
+  queries=$(add_parameter "$queries" "versionId=$version_id")
+fi
+canonical_request_data+=("$queries" "host:$host")
 if [ "$bypass_governance_retention" == "true" ]; then
  canonical_request_data+=("x-amz-bypass-governance-retention:true")
 fi
@@ -39,7 +46,7 @@ build_canonical_request "${canonical_request_data[@]}"
 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature

-curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key"
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?$queries"
 -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
 curl_command+=("${header_fields[@]}")
 curl_command+=(-o "$OUTPUT_FILE")
--- a/tests/rest_scripts/get_object_legal_hold.sh
+++ b/tests/rest_scripts/get_object_legal_hold.sh
@@ -29,7 +29,7 @@ current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")

 canonical_request_data=("GET" "/$bucket_name/$key")
 queries=""
-if [ "$VERSION_ID" != "" ]; then
+if [ "$version_id" != "" ]; then
  queries=$(add_parameter "$queries" "versionId=$version_id")
 fi
 queries=$(add_parameter "$queries" "legal-hold=")
@@ -39,7 +39,6 @@ if ! build_canonical_request "${canonical_request_data[@]}"; then
  log_rest 2 "error building request"
  exit 1
 fi
-echo -n "$canonical_request" > "cr.txt"

 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
--- a/tests/rest_scripts/put_object_legal_hold.sh
+++ b/tests/rest_scripts/put_object_legal_hold.sh
@@ -26,37 +26,40 @@ key="$(echo -n "$OBJECT_KEY" | jq -sRr @uri)"
 status="$STATUS"
 # shellcheck disable=SC2153
 omit_payload="${OMIT_PAYLOAD:=false}"
+# shellcheck disable=SC2153
+version_id="$VERSION_ID"

 if [ "$omit_payload" == "false" ]; then
  payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
 <LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
   <Status>$status</Status>
 </LegalHold>"
+else
+  payload=""
 fi

 payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
-content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64)
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")

-canonical_request="PUT
-/$bucket_name/$key
-legal-hold=
-content-md5:$content_md5
-host:$host
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-content-md5;host;x-amz-content-sha256;x-amz-date
-$payload_hash"
+canonical_request_data=("PUT" "/$bucket_name/$key")
+queries=""
+if [ "$version_id" != "" ]; then
+  queries=$(add_parameter "$queries" "versionId=$version_id")
+fi
+queries=$(add_parameter "$queries" "legal-hold=")
+canonical_request_data+=("$queries" "host:$host")
+canonical_request_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
+if ! build_canonical_request "${canonical_request_data[@]}"; then
+  log_rest 2 "error building request"
+  exit 1
+fi

 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature

-curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key?legal-hold="
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
-H "\"Content-MD5: $content_md5\""
-H "\"x-amz-content-sha256: $payload_hash\""
-H "\"x-amz-date: $current_date_time\"")
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key?$queries"
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
 if [ "$omit_payload" == "false" ]; then
  curl_command+=(-d "\"${payload//\"/\\\"}\"")
 fi
--- a/tests/rest_scripts/rest.sh
+++ b/tests/rest_scripts/rest.sh
@@ -95,7 +95,7 @@ log_rest() {
    return 1
  fi
  if [ "$BATS_TEST_NAME" != "" ]; then
-    log "$1" "$2"
+    log_with_stack_ref "$1" "$2" 2
  else
    echo "$2"
  fi
--- a/tests/test_common.sh
+++ b/tests/test_common.sh
@@ -42,7 +42,7 @@ source ./tests/drivers/drivers.sh
 # param:  command type
 # fail on test failure
 test_common_multipart_upload() {
-  run check_param_count "test_common_multipart_upload" "client type" 1 "$#"
+  run assert_param_count "test_common_multipart_upload" "client type" 1 "$#"
  assert_success

  bucket_file="largefile"
@@ -74,7 +74,7 @@ test_common_create_delete_bucket() {
    return
  fi

-  run check_param_count "test_common_create_delete_bucket" "client type" 1 "$#"
+  run assert_param_count "test_common_create_delete_bucket" "client type" 1 "$#"
  assert_success

  run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"
@@ -91,7 +91,7 @@ test_common_create_delete_bucket() {
 }

 test_common_copy_object() {
-  run check_param_count "test_common_copy_object" "client type" 1 "$#"
+  run assert_param_count "test_common_copy_object" "client type" 1 "$#"
  assert_success

  local object_name="test-object"
@@ -122,7 +122,7 @@ test_common_copy_object() {
 # param:  client
 # fail on error
 test_common_put_object_with_data() {
-  run check_param_count "test_common_put_object_with_data" "client type" 1 "$#"
+  run assert_param_count "test_common_put_object_with_data" "client type" 1 "$#"
  assert_success

  local object_name="test-object"
@@ -135,7 +135,7 @@ test_common_put_object_with_data() {
 # param:  client
 # fail on error
 test_common_put_object_no_data() {
-  run check_param_count "test_common_put_object_no_data" "client type" 1 "$#"
+  run assert_param_count "test_common_put_object_no_data" "client type" 1 "$#"
  assert_success

  local object_name="test-object"
@@ -148,7 +148,7 @@ test_common_put_object_no_data() {
 # params:  client, filename
 # fail on test failure
 test_common_put_object() {
-  run check_param_count "test_common_put_object" "client type, file" 2 "$#"
+  run assert_param_count "test_common_put_object" "client type, file" 2 "$#"
  assert_success

  run setup_bucket "$BUCKET_ONE_NAME"
@@ -179,7 +179,7 @@ test_common_put_object() {
 }

 test_common_put_get_object() {
-  run check_param_count "test_common_put_get_object" "client type" 1 "$#"
+  run assert_param_count "test_common_put_get_object" "client type" 1 "$#"
  assert_success

  local object_name="test-object"
@@ -204,7 +204,7 @@ test_common_put_get_object() {
 # param:  "aws" or "s3cmd"
 # pass if buckets are properly listed, fail if not
 test_common_list_buckets() {
-  run check_param_count "test_common_list_buckets" "client type" 1 "$#"
+  run assert_param_count "test_common_list_buckets" "client type" 1 "$#"
  assert_success

  run setup_buckets "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
@@ -215,7 +215,7 @@ test_common_list_buckets() {
 }

 test_common_list_objects() {
-  run check_param_count "test_common_list_objects" "client type" 1 "$#"
+  run assert_param_count "test_common_list_objects" "client type" 1 "$#"
  assert_success

  object_one="test-file-one"
@@ -234,7 +234,7 @@ test_common_list_objects() {
 }

 test_common_set_get_delete_bucket_tags() {
-  run check_param_count "test_common_set_get_delete_bucket_tags" "client type" 1 "$#"
+  run assert_param_count "test_common_set_get_delete_bucket_tags" "client type" 1 "$#"
  assert_success

  local key="test_key"
@@ -260,7 +260,7 @@ test_common_set_get_delete_bucket_tags() {
 }

 test_common_set_get_object_tags() {
-  run check_param_count "test_common_set_get_object_tags" "client type" 1 "$#"
+  run assert_param_count "test_common_set_get_object_tags" "client type" 1 "$#"
  assert_success

  local bucket_file="bucket-file"
@@ -284,7 +284,7 @@ test_common_set_get_object_tags() {
 }

 test_common_presigned_url_utf8_chars() {
-  run check_param_count "test_common_presigned_url_utf8_chars" "client type" 1 "$#"
+  run assert_param_count "test_common_presigned_url_utf8_chars" "client type" 1 "$#"
  assert_success

  local bucket_file="my-$%^&*;"
@@ -309,7 +309,7 @@ test_common_presigned_url_utf8_chars() {
 }

 test_common_list_objects_file_count() {
-  run check_param_count "test_common_list_objects_file_count" "client type" 1 "$#"
+  run assert_param_count "test_common_list_objects_file_count" "client type" 1 "$#"
  assert_success

  run create_test_file_count 1001
@@ -326,7 +326,7 @@ test_common_list_objects_file_count() {
 }

 test_common_delete_object_tagging() {
-  run check_param_count "test_common_delete_object_tagging" "client type" 1 "$#"
+  run assert_param_count "test_common_delete_object_tagging" "client type" 1 "$#"
  assert_success

  bucket_file="bucket_file"
@@ -353,7 +353,7 @@ test_common_delete_object_tagging() {
 }

 test_common_get_bucket_location() {
-  run check_param_count "test_common_get_bucket_location" "client type" 1 "$#"
+  run assert_param_count "test_common_get_bucket_location" "client type" 1 "$#"
  assert_success

  run setup_bucket "$BUCKET_ONE_NAME"
@@ -364,7 +364,7 @@ test_common_get_bucket_location() {
 }

 test_common_get_put_delete_bucket_policy() {
-  run check_param_count "test_common_get_put_delete_bucket_policy" "client type" 1 "$#"
+  run assert_param_count "test_common_get_put_delete_bucket_policy" "client type" 1 "$#"
  assert_success

  policy_file="policy_file"
@@ -401,7 +401,7 @@ test_common_get_put_delete_bucket_policy() {
 }

 test_common_ls_directory_object() {
-  run check_param_count "test_common_ls_directory_object" "client type" 1 "$#"
+  run assert_param_count "test_common_ls_directory_object" "client type" 1 "$#"
  assert_success

  test_file="a"
--- a/tests/util/util_bucket.sh
+++ b/tests/util/util_bucket.sh
@@ -186,7 +186,7 @@ bucket_cleanup_if_bucket_exists() {
    return 0
  fi

-  if bucket_exists "$1"; then
+  if [ "$2" == "true" ] || bucket_exists "$1"; then
    if ! bucket_cleanup "$1"; then
      log 2 "error deleting bucket and/or contents"
      return 1
--- a/tests/util/util_legal_hold.sh
+++ b/tests/util/util_legal_hold.sh
@@ -78,7 +78,7 @@ check_remove_legal_hold_versions() {
  #  return 1
  #fi
  if [ "$status" == "ON" ]; then
-    if ! put_object_legal_hold_version_id "$1" "$2" "$3" "OFF"; then
+    if ! put_object_legal_hold_rest_version_id "$1" "$2" "$3" "OFF"; then
      log 2 "error removing legal hold of version ID"
      return 1
    fi
--- a/tests/util/util_lock_config.sh
+++ b/tests/util/util_lock_config.sh
@@ -112,10 +112,6 @@ check_object_lock_config_enabled_rest() {
  if ! check_param_count "check_object_lock_config_enabled_rest" "bucket" 1 $#; then
    return 1
  fi
-  if [ $# -ne 1 ]; then
-    log 2 "'get_check_object_lock_config_rest' requires bucket name"
-    return 1
-  fi
  if ! get_object_lock_configuration_rest "$1"; then
    log 2 "error getting object lock config"
    return 1
--- a/tests/util/util_versioning.sh
+++ b/tests/util/util_versioning.sh
@@ -2,6 +2,7 @@

 source ./tests/commands/get_bucket_versioning.sh
 source ./tests/commands/list_object_versions.sh
+source ./tests/drivers/drivers.sh

 check_if_versioning_enabled() {
  if ! check_param_count "check_if_versioning_enabled" "bucket" 1 $#; then
@@ -61,15 +62,12 @@ delete_object_version_with_or_without_retention() {
    if ! check_remove_legal_hold_versions "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
      log 2 "error checking, removing legal hold versions"
    fi
-    if ! put_object_legal_hold_version_id "$1" "${version_keys[$idx]}" "${version_ids[$idx]}" "OFF"; then
-      log 2 "error turning off object legal hold"
-    fi
-    if ! delete_object_version_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
+    if ! delete_object_version_rest_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
      log 2 "error deleting object version, bypassing retention"
      return 1
    fi
  else
-    if ! delete_object_version "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
+    if ! delete_object_version_rest "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
      log 2 "error deleting object version"
      return 1
    fi