diff --git a/tests/commands/delete_object.sh b/tests/commands/delete_object.sh index ce94b54..cd02396 100644 --- a/tests/commands/delete_object.sh +++ b/tests/commands/delete_object.sh @@ -65,6 +65,23 @@ delete_object_bypass_retention() { return 0 } +# shellcheck disable=SC2317 +delete_object_rest() { + if ! check_param_count "delete_object_rest" "bucket, key" 2 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then + log 2 "error deleting object: $result" + return 1 + fi + if [ "$result" != "204" ]; then + delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt") + log 2 "expected '204', was '$result' ($delete_object_error)" + return 1 + fi + return 0 +} + delete_object_version() { if ! check_param_count "delete_object_version" "bucket, key, version ID" 3 $#; then return 1 @@ -76,6 +93,22 @@ delete_object_version() { return 0 } +delete_object_version_rest() { + if ! check_param_count "delete_object_version_rest" "bucket name, object name, version ID" 3 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then + log 2 "error deleting object: $result" + return 1 + fi + if [ "$result" != "204" ]; then + delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt") + log 2 "expected '204', was '$result' ($delete_object_error)" + return 1 + fi + return 0 +} + delete_object_version_bypass_retention() { if ! check_param_count "delete_object_version_bypass_retention" "bucket, key, version ID" 3 $#; then return 1 @@ -87,6 +120,22 @@ delete_object_version_bypass_retention() { return 0 } +delete_object_version_rest_bypass_retention() { + if ! check_param_count "delete_object_version_rest_bypass_retention" "bucket, key, version ID" 3 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" BYPASS_GOVERNANCE_RETENTION="true" \ + OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then + log 2 "error deleting object: $result" + return 1 + fi + if [ "$result" != "204" ]; then + log 2 "expected '204', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" + return 1 + fi + return 0 +} + delete_object_with_user() { record_command "delete-object" "client:$1" if ! check_param_count "delete_object_version_bypass_retention" "command type, bucket, key, access ID, secret key" 5 $#; then @@ -124,4 +173,4 @@ delete_object_rest() { return 1 fi return 0 -} \ No newline at end of file +} diff --git a/tests/commands/put_object_legal_hold.sh b/tests/commands/put_object_legal_hold.sh index f2fec64..5bf7191 100644 --- a/tests/commands/put_object_legal_hold.sh +++ b/tests/commands/put_object_legal_hold.sh @@ -60,3 +60,18 @@ put_object_legal_hold_version_id() { fi return 0 } + +put_object_legal_hold_rest_version_id() { + if ! check_param_count "put_object_legal_hold_rest" "bucket, key, version ID, hold status" 4 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$3" STATUS="$4" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_legal_hold.sh 2>&1); then + log 2 "error putting object legal hold: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" + return 1 + fi + return 0 +} diff --git a/tests/drivers/drivers.sh b/tests/drivers/drivers.sh index e333c9b..96abc13 100644 --- a/tests/drivers/drivers.sh +++ b/tests/drivers/drivers.sh @@ -20,7 +20,19 @@ check_param_count() { return 1 fi if [ "$3" -ne "$4" ]; then - log 2 "function $1 requires $2" + log_with_stack_ref 2 "function $1 requires $2" 2 + return 1 + fi + return 0 +} + +assert_param_count() { + if [ $# -ne 4 ]; then + log 2 "'assert_param_count' requires function name, params list, expected, actual" + return 1 + fi + if [ "$3" -ne "$4" ]; then + log_with_stack_ref 2 "function $1 requires $2" 4 return 1 fi return 0 @@ -32,7 +44,7 @@ check_param_count_gt() { return 1 fi if [ "$3" -gt "$4" ]; then - log 2 "function $1 requires $2" + log_with_stack_ref 2 "function $1 requires $2" 2 return 1 fi return 0 diff --git a/tests/logger.sh b/tests/logger.sh index da50f34..e6a0d77 100644 --- a/tests/logger.sh +++ b/tests/logger.sh @@ -16,9 +16,31 @@ # levels: 1 - crit, 2 - err, 3 - warn, 4 - info, 5 - debug, 6 - trace +check_log_params() { + if [ $# -ne 4 ]; then + echo "'check_log_params' requires function name, params list, expected, actual" + return 1 + fi + if [ "$3" -ne "$4" ]; then + echo "function $1 requires $2" 2 + return 1 + fi + return 0 +} + log() { - if [[ $# -ne 2 ]]; then - echo "log function requires level, message" + if ! check_log_params "log" "level, message" 2 $#; then + return 1 + fi + if ! log_with_stack_ref "$1" "$2" 2; then + echo "error logging with stack ref" + return 1 + fi + return 0 +} + +log_with_stack_ref() { + if ! check_log_params "log_with_stack_ref" "level, message, stack reference" 3 $#; then return 1 fi # shellcheck disable=SC2153 @@ -36,15 +58,14 @@ log() { *) echo "invalid log level $1"; return 1 esac if [[ ( "$2" == *"access"* ) || ( "$2" == *"secret"* ) || ( "$2" == *"Credential="* ) ]]; then - log_mask "$log_level" "$2" + log_mask "$log_level" "$2" "$3" return 0 fi - log_message "$log_level" "$2" + log_message "$log_level" "$2" "$3" } log_mask() { - if [[ $# -ne 2 ]]; then - echo "mask and log requires level, string" + if ! check_log_params "log_mask" "level, string, stack reference" 3 $#; then return 1 fi @@ -53,12 +74,11 @@ log_mask() { return 1 fi - log_message "$log_level" "$masked_data" + log_message "$log_level" "$masked_data" "$3" } mask_args() { - if [ $# -ne 1 ]; then - echo "'mask_args' requires string" + if ! check_log_params "mask_args" "string" 1 $#; then return 1 fi unmasked_array=() @@ -101,8 +121,7 @@ mask_arg_array() { } check_arg_for_mask() { - if [ $# -ne 1 ]; then - echo "'check_arg_for_mask' requires arg" + if ! check_log_params "check_arg_for_mask" "arg" 1 $#; then return 1 fi if [[ $mask_next == true ]]; then @@ -137,16 +156,16 @@ check_arg_for_mask() { } log_message() { - if [[ $# -ne 2 ]]; then - echo "log message requires level, message" + if ! check_log_params "log_message" "level, message, stack reference" 3 $#; then return 1 fi + local bash_source_ref=$(($3+1)) now="$(date "+%Y-%m-%d %H:%M:%S")" if [[ ( "$1" == "CRIT" ) || ( "$1" == "ERROR" ) ]]; then echo "$now $1 $2" >&2 fi if [[ -n "$TEST_LOG_FILE" ]]; then - echo "$now ${BASH_SOURCE[2]}:${BASH_LINENO[1]} $1 $2" >> "$TEST_LOG_FILE.tmp" + echo "$now ${BASH_SOURCE[$bash_source_ref]}:${BASH_LINENO[$3]} $1 $2" >> "$TEST_LOG_FILE.tmp" fi sync } diff --git a/tests/rest_scripts/delete_object.sh b/tests/rest_scripts/delete_object.sh index 8eae047..652e356 100755 --- a/tests/rest_scripts/delete_object.sh +++ b/tests/rest_scripts/delete_object.sh @@ -24,11 +24,18 @@ bucket_name="$BUCKET_NAME" key=$(echo -n "$OBJECT_KEY" | jq -sRr 'split("/") | map(@uri) | join("/")') # shellcheck disable=SC2154 bypass_governance_retention=${BYPASS_GOVERNANCE_RETENTION:=false} +# shellcheck disable=SC2153,SC2154 +version_id="$VERSION_ID" current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") #x-amz-object-attributes:ETag -canonical_request_data+=("DELETE" "/$bucket_name/$key" "" "host:$host") +canonical_request_data=("DELETE" "/$bucket_name/$key") +queries="" +if [ "$version_id" != "" ]; then + queries=$(add_parameter "$queries" "versionId=$version_id") +fi +canonical_request_data+=("$queries" "host:$host") if [ "$bypass_governance_retention" == "true" ]; then canonical_request_data+=("x-amz-bypass-governance-retention:true") fi @@ -39,7 +46,7 @@ build_canonical_request "${canonical_request_data[@]}" # shellcheck disable=SC2119 create_canonical_hash_sts_and_signature -curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key" +curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?$queries" -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") curl_command+=("${header_fields[@]}") curl_command+=(-o "$OUTPUT_FILE") diff --git a/tests/rest_scripts/get_object_legal_hold.sh b/tests/rest_scripts/get_object_legal_hold.sh index 21ad2ab..12345b5 100755 --- a/tests/rest_scripts/get_object_legal_hold.sh +++ b/tests/rest_scripts/get_object_legal_hold.sh @@ -29,7 +29,7 @@ current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") canonical_request_data=("GET" "/$bucket_name/$key") queries="" -if [ "$VERSION_ID" != "" ]; then +if [ "$version_id" != "" ]; then queries=$(add_parameter "$queries" "versionId=$version_id") fi queries=$(add_parameter "$queries" "legal-hold=") @@ -39,7 +39,6 @@ if ! build_canonical_request "${canonical_request_data[@]}"; then log_rest 2 "error building request" exit 1 fi -echo -n "$canonical_request" > "cr.txt" # shellcheck disable=SC2119 create_canonical_hash_sts_and_signature diff --git a/tests/rest_scripts/put_object_legal_hold.sh b/tests/rest_scripts/put_object_legal_hold.sh index 9081888..b6385c7 100755 --- a/tests/rest_scripts/put_object_legal_hold.sh +++ b/tests/rest_scripts/put_object_legal_hold.sh @@ -26,37 +26,40 @@ key="$(echo -n "$OBJECT_KEY" | jq -sRr @uri)" status="$STATUS" # shellcheck disable=SC2153 omit_payload="${OMIT_PAYLOAD:=false}" +# shellcheck disable=SC2153 +version_id="$VERSION_ID" if [ "$omit_payload" == "false" ]; then payload=" $status " +else + payload="" fi payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" -content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64) current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") -canonical_request="PUT -/$bucket_name/$key -legal-hold= -content-md5:$content_md5 -host:$host -x-amz-content-sha256:$payload_hash -x-amz-date:$current_date_time - -content-md5;host;x-amz-content-sha256;x-amz-date -$payload_hash" +canonical_request_data=("PUT" "/$bucket_name/$key") +queries="" +if [ "$version_id" != "" ]; then + queries=$(add_parameter "$queries" "versionId=$version_id") +fi +queries=$(add_parameter "$queries" "legal-hold=") +canonical_request_data+=("$queries" "host:$host") +canonical_request_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time") +if ! build_canonical_request "${canonical_request_data[@]}"; then + log_rest 2 "error building request" + exit 1 +fi # shellcheck disable=SC2119 create_canonical_hash_sts_and_signature -curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key?legal-hold=" --H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" --H "\"Content-MD5: $content_md5\"" --H "\"x-amz-content-sha256: $payload_hash\"" --H "\"x-amz-date: $current_date_time\"") +curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key?$queries" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") +curl_command+=("${header_fields[@]}") if [ "$omit_payload" == "false" ]; then curl_command+=(-d "\"${payload//\"/\\\"}\"") fi diff --git a/tests/rest_scripts/rest.sh b/tests/rest_scripts/rest.sh index 41d59fd..789eacb 100644 --- a/tests/rest_scripts/rest.sh +++ b/tests/rest_scripts/rest.sh @@ -95,7 +95,7 @@ log_rest() { return 1 fi if [ "$BATS_TEST_NAME" != "" ]; then - log "$1" "$2" + log_with_stack_ref "$1" "$2" 2 else echo "$2" fi diff --git a/tests/test_common.sh b/tests/test_common.sh index b8ae132..446aad5 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -42,7 +42,7 @@ source ./tests/drivers/drivers.sh # param: command type # fail on test failure test_common_multipart_upload() { - run check_param_count "test_common_multipart_upload" "client type" 1 "$#" + run assert_param_count "test_common_multipart_upload" "client type" 1 "$#" assert_success bucket_file="largefile" @@ -74,7 +74,7 @@ test_common_create_delete_bucket() { return fi - run check_param_count "test_common_create_delete_bucket" "client type" 1 "$#" + run assert_param_count "test_common_create_delete_bucket" "client type" 1 "$#" assert_success run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME" @@ -91,7 +91,7 @@ test_common_create_delete_bucket() { } test_common_copy_object() { - run check_param_count "test_common_copy_object" "client type" 1 "$#" + run assert_param_count "test_common_copy_object" "client type" 1 "$#" assert_success local object_name="test-object" @@ -122,7 +122,7 @@ test_common_copy_object() { # param: client # fail on error test_common_put_object_with_data() { - run check_param_count "test_common_put_object_with_data" "client type" 1 "$#" + run assert_param_count "test_common_put_object_with_data" "client type" 1 "$#" assert_success local object_name="test-object" @@ -135,7 +135,7 @@ test_common_put_object_with_data() { # param: client # fail on error test_common_put_object_no_data() { - run check_param_count "test_common_put_object_no_data" "client type" 1 "$#" + run assert_param_count "test_common_put_object_no_data" "client type" 1 "$#" assert_success local object_name="test-object" @@ -148,7 +148,7 @@ test_common_put_object_no_data() { # params: client, filename # fail on test failure test_common_put_object() { - run check_param_count "test_common_put_object" "client type, file" 2 "$#" + run assert_param_count "test_common_put_object" "client type, file" 2 "$#" assert_success run setup_bucket "$BUCKET_ONE_NAME" @@ -179,7 +179,7 @@ test_common_put_object() { } test_common_put_get_object() { - run check_param_count "test_common_put_get_object" "client type" 1 "$#" + run assert_param_count "test_common_put_get_object" "client type" 1 "$#" assert_success local object_name="test-object" @@ -204,7 +204,7 @@ test_common_put_get_object() { # param: "aws" or "s3cmd" # pass if buckets are properly listed, fail if not test_common_list_buckets() { - run check_param_count "test_common_list_buckets" "client type" 1 "$#" + run assert_param_count "test_common_list_buckets" "client type" 1 "$#" assert_success run setup_buckets "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" @@ -215,7 +215,7 @@ test_common_list_buckets() { } test_common_list_objects() { - run check_param_count "test_common_list_objects" "client type" 1 "$#" + run assert_param_count "test_common_list_objects" "client type" 1 "$#" assert_success object_one="test-file-one" @@ -234,7 +234,7 @@ test_common_list_objects() { } test_common_set_get_delete_bucket_tags() { - run check_param_count "test_common_set_get_delete_bucket_tags" "client type" 1 "$#" + run assert_param_count "test_common_set_get_delete_bucket_tags" "client type" 1 "$#" assert_success local key="test_key" @@ -260,7 +260,7 @@ test_common_set_get_delete_bucket_tags() { } test_common_set_get_object_tags() { - run check_param_count "test_common_set_get_object_tags" "client type" 1 "$#" + run assert_param_count "test_common_set_get_object_tags" "client type" 1 "$#" assert_success local bucket_file="bucket-file" @@ -284,7 +284,7 @@ test_common_set_get_object_tags() { } test_common_presigned_url_utf8_chars() { - run check_param_count "test_common_presigned_url_utf8_chars" "client type" 1 "$#" + run assert_param_count "test_common_presigned_url_utf8_chars" "client type" 1 "$#" assert_success local bucket_file="my-$%^&*;" @@ -309,7 +309,7 @@ test_common_presigned_url_utf8_chars() { } test_common_list_objects_file_count() { - run check_param_count "test_common_list_objects_file_count" "client type" 1 "$#" + run assert_param_count "test_common_list_objects_file_count" "client type" 1 "$#" assert_success run create_test_file_count 1001 @@ -326,7 +326,7 @@ test_common_list_objects_file_count() { } test_common_delete_object_tagging() { - run check_param_count "test_common_delete_object_tagging" "client type" 1 "$#" + run assert_param_count "test_common_delete_object_tagging" "client type" 1 "$#" assert_success bucket_file="bucket_file" @@ -353,7 +353,7 @@ test_common_delete_object_tagging() { } test_common_get_bucket_location() { - run check_param_count "test_common_get_bucket_location" "client type" 1 "$#" + run assert_param_count "test_common_get_bucket_location" "client type" 1 "$#" assert_success run setup_bucket "$BUCKET_ONE_NAME" @@ -364,7 +364,7 @@ test_common_get_bucket_location() { } test_common_get_put_delete_bucket_policy() { - run check_param_count "test_common_get_put_delete_bucket_policy" "client type" 1 "$#" + run assert_param_count "test_common_get_put_delete_bucket_policy" "client type" 1 "$#" assert_success policy_file="policy_file" @@ -401,7 +401,7 @@ test_common_get_put_delete_bucket_policy() { } test_common_ls_directory_object() { - run check_param_count "test_common_ls_directory_object" "client type" 1 "$#" + run assert_param_count "test_common_ls_directory_object" "client type" 1 "$#" assert_success test_file="a" diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index 5348c98..5dddf70 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -186,7 +186,7 @@ bucket_cleanup_if_bucket_exists() { return 0 fi - if bucket_exists "$1"; then + if [ "$2" == "true" ] || bucket_exists "$1"; then if ! bucket_cleanup "$1"; then log 2 "error deleting bucket and/or contents" return 1 diff --git a/tests/util/util_legal_hold.sh b/tests/util/util_legal_hold.sh index 618ed16..c0559a8 100644 --- a/tests/util/util_legal_hold.sh +++ b/tests/util/util_legal_hold.sh @@ -78,7 +78,7 @@ check_remove_legal_hold_versions() { # return 1 #fi if [ "$status" == "ON" ]; then - if ! put_object_legal_hold_version_id "$1" "$2" "$3" "OFF"; then + if ! put_object_legal_hold_rest_version_id "$1" "$2" "$3" "OFF"; then log 2 "error removing legal hold of version ID" return 1 fi diff --git a/tests/util/util_lock_config.sh b/tests/util/util_lock_config.sh index faf2b4a..767c12b 100644 --- a/tests/util/util_lock_config.sh +++ b/tests/util/util_lock_config.sh @@ -112,10 +112,6 @@ check_object_lock_config_enabled_rest() { if ! check_param_count "check_object_lock_config_enabled_rest" "bucket" 1 $#; then return 1 fi - if [ $# -ne 1 ]; then - log 2 "'get_check_object_lock_config_rest' requires bucket name" - return 1 - fi if ! get_object_lock_configuration_rest "$1"; then log 2 "error getting object lock config" return 1 diff --git a/tests/util/util_versioning.sh b/tests/util/util_versioning.sh index 2e37459..12cb40f 100644 --- a/tests/util/util_versioning.sh +++ b/tests/util/util_versioning.sh @@ -2,6 +2,7 @@ source ./tests/commands/get_bucket_versioning.sh source ./tests/commands/list_object_versions.sh +source ./tests/drivers/drivers.sh check_if_versioning_enabled() { if ! check_param_count "check_if_versioning_enabled" "bucket" 1 $#; then @@ -61,15 +62,12 @@ delete_object_version_with_or_without_retention() { if ! check_remove_legal_hold_versions "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then log 2 "error checking, removing legal hold versions" fi - if ! put_object_legal_hold_version_id "$1" "${version_keys[$idx]}" "${version_ids[$idx]}" "OFF"; then - log 2 "error turning off object legal hold" - fi - if ! delete_object_version_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + if ! delete_object_version_rest_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then log 2 "error deleting object version, bypassing retention" return 1 fi else - if ! delete_object_version "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + if ! delete_object_version_rest "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then log 2 "error deleting object version" return 1 fi