From 1632c8c006bd4c43a240ce3d9bf979775a6d068b Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Tue, 17 Mar 2026 15:27:24 -0300 Subject: [PATCH] test: ListObjectsV2 query updates, file updates, bash cleanup --- tests/commands/create_bucket.sh | 3 +- tests/commands/head_object.sh | 5 +- tests/commands/list_buckets.sh | 7 +- tests/commands/upload_part_copy.sh | 2 +- .../complete_multipart_upload_rest.sh | 6 +- .../delete_bucket/delete_bucket_rest.sh | 8 +- .../delete_object/delete_object_rest.sh | 32 +++- tests/drivers/file.sh | 150 ++++++++++++--- .../get_object_tagging_rest.sh | 6 +- tests/drivers/head_object/head_object_rest.sh | 21 +- .../drivers/list_buckets/list_buckets_rest.sh | 85 +++++++-- .../list_object_versions_rest.sh | 74 +++++--- .../drivers/list_objects/list_objects_rest.sh | 179 +++++++++++++++++- tests/drivers/params.sh | 12 ++ .../put_object_tagging_rest.sh | 8 +- tests/drivers/rest.sh | 80 +++++--- tests/drivers/upload_part/upload_part_rest.sh | 24 ++- tests/drivers/xml.sh | 16 +- tests/rest_scripts/command/canonicalQuery.go | 41 ++-- tests/test_common.sh | 3 +- tests/test_rest_list_objects.sh | 143 ++++++++++++-- tests/test_s3api_multipart.sh | 42 ++-- tests/util/util_list_objects.sh | 25 ++- tests/util/util_multipart.sh | 21 +- tests/util/util_multipart_abort.sh | 7 +- .../util/util_multipart_before_completion.sh | 33 +++- tests/util/util_versioning.sh | 61 +++--- 27 files changed, 844 insertions(+), 250 deletions(-) diff --git a/tests/commands/create_bucket.sh b/tests/commands/create_bucket.sh index 43d85096..8a786c1b 100644 --- a/tests/commands/create_bucket.sh +++ b/tests/commands/create_bucket.sh @@ -25,8 +25,7 @@ create_bucket() { return 1 fi - local exit_code=0 - local error + local exit_code=0 error if [[ $1 == 's3' ]]; then error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == 's3api' ]]; then diff --git a/tests/commands/head_object.sh b/tests/commands/head_object.sh index 53363d0a..976d4bfb 100644 --- a/tests/commands/head_object.sh +++ b/tests/commands/head_object.sh @@ -59,10 +59,11 @@ head_object_rest_expect_success_callback() { return 1 fi env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2 $3" - if ! send_rest_command_expect_success_callback "$env_vars" "./tests/rest_scripts/head_object.sh" "200" "$4"; then - log 2 "error sending REST command and checking error" + if ! callback_result=$(send_rest_command_expect_success_callback "$env_vars" "./tests/rest_scripts/head_object.sh" "200" "$4" 2>&1); then + log 2 "error sending REST command and checking error: $callback_result" return 1 fi + echo "$callback_result" return 0 } diff --git a/tests/commands/list_buckets.sh b/tests/commands/list_buckets.sh index d644bd90..2157d38a 100644 --- a/tests/commands/list_buckets.sh +++ b/tests/commands/list_buckets.sh @@ -118,9 +118,12 @@ list_buckets_rest() { if ! check_param_count_v2 "params, callback" 2 $#; then return 1 fi - if ! send_rest_command_expect_success_callback "$1" "./tests/rest_scripts/list_buckets.sh" "200" "$2"; then - log 2 "error sending REST command and checking error" + if ! callback_result=$(send_rest_command_expect_success_callback "$1" "./tests/rest_scripts/list_buckets.sh" "200" "$2" 2>&1); then + log 2 "error sending REST command and checking error: $callback_result" return 1 + else + buckets="$callback_result" fi + echo "$buckets" return 0 } diff --git a/tests/commands/upload_part_copy.sh b/tests/commands/upload_part_copy.sh index f6a638cd..6cbcacf5 100644 --- a/tests/commands/upload_part_copy.sh +++ b/tests/commands/upload_part_copy.sh @@ -20,7 +20,7 @@ upload_part_copy() { return 1 fi local etag_json - log 5 "parameters: $1 $2 $3 $4 $5" + log 5 "parameters: '$1' '$2' '$3' '$4' '$5'" etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$? if [[ $uploaded -ne 0 ]]; then log 2 "Error uploading part $5: $etag_json" diff --git a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh index 9c4d39d1..8b77ab1c 100644 --- a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh +++ b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh @@ -193,10 +193,12 @@ complete_multipart_upload_invalid_object_size_string() { if ! check_param_count_v2 "bucket, key, file" 3 $#; then return 1 fi - if ! multipart_upload_rest_before_completion "$1" "$2" "$3" 2; then - log 2 "error performing multipart upload before completion" + local response upload_id parts_payload + if ! response=$(multipart_upload_rest_before_completion "$1" "$2" "$3" 2 2>&1); then + log 2 "error performing multipart upload before completion: $response" return 1 fi + read -r upload_id parts_payload <<< "$response" if ! complete_multipart_upload_rest_expect_error "$1" "$2" "$upload_id" "$parts_payload" "MULTIPART_OBJECT_SIZE=size" "400" "InvalidRequest" "Value for x-amz-mp-object-size header is invalid"; then log 2 "error completing multipart upload" return 1 diff --git a/tests/drivers/delete_bucket/delete_bucket_rest.sh b/tests/drivers/delete_bucket/delete_bucket_rest.sh index 25cb89f2..ff455bdd 100644 --- a/tests/drivers/delete_bucket/delete_bucket_rest.sh +++ b/tests/drivers/delete_bucket/delete_bucket_rest.sh @@ -29,12 +29,12 @@ delete_buckets_with_prefix() { log 2 "delete_buckets_with_prefix requires non-empty prefix" return 1 fi - if ! list_buckets_rest "PREFIX=$1" "parse_bucket_list"; then - log 2 "error listing buckets with prefix" + local response + if ! response=$(list_buckets_rest "PREFIX=$1" "parse_bucket_list" 2>&1); then + log 2 "error listing buckets with prefix: $response" return 1 fi - # shellcheck disable=SC2154 - log 5 "buckets: ${bucket_array[*]}" + read -r -a bucket_array <<< "$response" for bucket in "${bucket_array[@]}"; do if ! delete_bucket_recursive "$bucket"; then log 2 "error with recursive bucket delete of bucket '$bucket'" diff --git a/tests/drivers/delete_object/delete_object_rest.sh b/tests/drivers/delete_object/delete_object_rest.sh index 49f38410..5322c794 100644 --- a/tests/drivers/delete_object/delete_object_rest.sh +++ b/tests/drivers/delete_object/delete_object_rest.sh @@ -24,13 +24,16 @@ list_and_delete_objects() { if ! check_param_count "list_and_delete_objects" "bucket" 1 $#; then return 1 fi - if ! list_objects_rest "$1" "parse_objects_list_rest"; then - log 2 "error getting object list" + local response + if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then + log 2 "error getting object list: $response" return 1 fi - # shellcheck disable=SC2154 - log 5 "objects: ${object_array[*]}" + mapfile -t object_array <<< "$response" for object in "${object_array[@]}"; do + if [ "$object" == "" ]; then + break + fi if ! clear_object_in_bucket "$1" "$object"; then log 2 "error deleting object $object" return 1 @@ -114,11 +117,16 @@ put_object_with_lock_mode_and_delete_latest_version() { log 2 "error sending put object command with object lock" return 1 fi - if ! send_rest_go_command_callback "200" "parse_latest_version_id" \ - "-method" "GET" "-bucketName" "$2" "-query" "versions="; then - log 2 "error checking versions before deletion" + + local response + if ! response=$(send_rest_go_command_callback "200" "parse_latest_version_id" \ + "-method" "GET" "-bucketName" "$2" "-query" "versions=" 2>&1); then + log 2 "error checking versions before deletion: $response" return 1 + else + version_id="$response" fi + log 5 "version ID: $version_id" if ! delete_object_version_rest_expect_error "$2" "$3" "$version_id" "403" "AccessDenied" "object protected by object lock"; then log 2 "shouldn't have been able to delete" return 1 @@ -140,11 +148,15 @@ attempt_to_delete_version_after_retention_policy() { log 2 "error sending put object command" return 1 fi - if ! send_rest_go_command_callback "200" "parse_latest_version_id" \ - "-method" "GET" "-bucketName" "$2" "-query" "versions="; then - log 2 "error checking versions before deletion" + + local response + if ! response=$(send_rest_go_command_callback "200" "parse_latest_version_id" \ + "-method" "GET" "-bucketName" "$2" "-query" "versions=" 2>&1); then + log 2 "error checking versions before deletion: $response" return 1 fi + + version_id="$response" if ! delete_object_version_rest_expect_error "$2" "$3" "$version_id" "403" "AccessDenied" "object protected by object lock"; then log 2 "shouldn't have been able to delete" return 1 diff --git a/tests/drivers/file.sh b/tests/drivers/file.sh index e1cb38b0..e645868c 100644 --- a/tests/drivers/file.sh +++ b/tests/drivers/file.sh @@ -163,15 +163,25 @@ setup_bucket_and_large_file_v2() { } setup_bucket_and_large_file_v3() { - if ! check_param_count_v2 "bucket env var" 1 $#; then + if ! check_param_count_ge_le "bucket env var, file size in MBs (optional)" 1 2 $#; then return 1 fi - if ! bucket_name=$(setup_bucket_v3 "$1" 2>&1); then - log 2 "error setting up bucket: $bucket_name" + + local response bucket_name file_name error + if ! response=$(setup_bucket_v3 "$1" 2>&1); then + log 2 "error setting up bucket: $response" return 1 fi - if ! file_name=$(create_large_file "$file_name" 2>&1); then - log 2 "error creating large file: $file_name" + bucket_name="$response" + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! error=$(create_large_file "$file_name" "$2" 2>&1); then + log 2 "error creating large file: $error" return 1 fi echo "$bucket_name $file_name" @@ -521,57 +531,139 @@ compare_files() { return 1 } -# Usage: create_large_file [filename] [size_in_mb] -# If filename is omitted, it generates one. Defaults to 160MB. +# generate 160MB file +# input: filename +# fail on error create_large_file() { - if ! check_param_count_le "filename (optional), size in MB (optional)" 2 $#; then + log 6 "create_large_file" + if ! check_param_count_ge_le "file name, size in MBs (optional)" 1 2 $#; then return 1 fi - - local file_name="$1" - local size_mb="${2:-160}" - local error - - if [ -z "$TEST_FILE_FOLDER" ]; then - log 2 "TEST_FILE_FOLDER must be defined" + file_size=${2:-160} + if ! error=$(create_large_file_with_size "$1" "$file_size" 2>&1); then + log 2 "error creating 160MB file: $error" return 1 fi + return 0 +} - if [[ -z "$file_name" ]]; then - if ! file_name=$(get_file_name 2>&1); then - log 2 "error generating automatic file name: $file_name" +create_large_file_v2() { + local response file_name + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! error=$(create_large_file_with_size "$file_name" 160 2>&1); then + log 2 "error creating 160MB file with name '$file_name': $error" + return 1 + fi + echo "$file_name" + return 0 +} + +create_large_file_with_size() { + if ! check_param_count_v2 "file name, size in MB" 2 $#; then + return 1 + fi + file_size=$(($2*1024*1024)) + if ! error=$(create_test_file_base "$1" "$file_size" 2>&1); then + log 2 "error creating large file with size ${2}MB: $error" + return 1 + fi + return 0 +} + +create_test_file_base() { + if ! check_param_count_ge_le "file name, size (optional), block size (optional)" 1 3 $#; then + return 1 + fi + log 5 "params: $*" + + local file_name=$1 + local requested_size=${2:-10} + local block_size=${3:-1048576} + + if [ "$file_name" == "" ]; then + log 2 "no file name specified" + return 1 + fi + local full_path="$TEST_FILE_FOLDER/$file_name" + + if [[ "$requested_size" -eq 0 ]]; then + touch "$full_path" + elif [[ "$requested_size" -lt "$block_size" ]]; then + # For small files, use the size as the block size (1 write) + if ! error=$(dd if=/dev/urandom of="$full_path" bs="$requested_size" count=1 conv=notrunc 2>&1); then + log 2 "error writing file: $error" + return 1 + fi + else + # For large files, use 1MB chunks to save RAM + count=$(( requested_size / block_size )) + remainder=$(( requested_size % block_size )) + + # Write the big 1MB chunks + if ! error=$(dd if=/dev/urandom of="$full_path" bs="$block_size" count="$count" conv=notrunc 2>&1); then + log 2 "error writing file chunk: $error" + return 1 + fi + + # If there's a remainder (e.g., 1.5MB), append the last few bytes + if [[ $remainder -gt 0 ]] && ! error=$(dd if=/dev/urandom of="$full_path" bs=1 count="$remainder" oflag=append conv=notrunc 2>&1); then + log 2 "error writing final file chunk: $error" return 1 fi fi - - log 6 "Creating ${size_mb}MB file: $file_name" - # bs=1M is significantly faster than bs=1024 for large files - if ! error=$(dd if=/dev/urandom of="${TEST_FILE_FOLDER}/${file_name}" bs=1M count="$size_mb" 2>&1); then - log 2 "error creating ${size_mb}MB file at ${file_name}: $error" - return 1 - fi - echo "$file_name" return 0 } +create_and_split_large_file() { +if ! check_param_count_v2 "file name, size in MB, pieces" 3 $#; then + return 1 +fi +if ! error=$(create_large_file_with_size "$1" "$2" 2>&1); then + log 2 "error creating large file: $error" + return 1 +fi +if ! split_file "$TEST_FILE_FOLDER/$1" "$3"; then + log 2 "error splitting file" + return 1 +fi +} + # param: number of files # fail on error create_test_file_count() { if ! check_param_count_v2 "number of files" 1 $#; then return 1 fi + if [ -z "$TEST_FILE_FOLDER" ]; then + log 2 "TEST_FILE_FOLDER not defined" + return 1 + fi + + local response file_prefix + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file prefix: $response" + return 1 + fi + file_prefix="$response" + for ((i=1;i<=$1;i++)) { - if ! error=$(touch "$TEST_FILE_FOLDER/file_$i" 2>&1); then + if ! error=$(touch "$TEST_FILE_FOLDER/${file_prefix}_${i}" 2>&1); then log 2 "error creating file_$i: $error" return 1 fi } # shellcheck disable=SC2153 if [[ $LOG_LEVEL -ge 5 ]]; then - ls_result=$(ls "$TEST_FILE_FOLDER/file_*") + ls_result=$(ls "$TEST_FILE_FOLDER/${file_prefix}_*" 2>&1) log 5 "$ls_result" fi + echo "$file_prefix" return 0 } @@ -579,7 +671,7 @@ download_and_compare_file_with_user() { if ! check_param_count_gt "original file, bucket, key, destination, username, password, chunk size (optional)" 6 $#; then return 1 fi - if [ -e "$4" ] && ! error=$(rm -f "$4"); then + if [ -e "$4" ] && ! error=$(rm -f "$4" 2>&1); then log 2 "error deleting local file at download destination before download: $error" return 1 fi diff --git a/tests/drivers/get_object_tagging/get_object_tagging_rest.sh b/tests/drivers/get_object_tagging/get_object_tagging_rest.sh index 14c896b1..cdac7638 100644 --- a/tests/drivers/get_object_tagging/get_object_tagging_rest.sh +++ b/tests/drivers/get_object_tagging/get_object_tagging_rest.sh @@ -88,10 +88,10 @@ get_check_object_tags_empty() { } check_header_version_id() { - if ! check_param_count_v2 "data file" 1 $#; then + if ! check_param_count_v2 "data file, version ID" 2 $#; then return 1 fi - if ! check_for_header_key_and_value "$1" "x-amz-version-id" "$version_id"; then + if ! check_for_header_key_and_value "$1" "x-amz-version-id" "$2"; then log 2 "error checking for x-amz-version-id header" return 1 fi @@ -107,7 +107,7 @@ add_version_tags_check_version_id() { return 1 fi if ! send_rest_go_command_callback "200" "check_header_version_id" "-bucketName" "$1" "-objectKey" "$2" "-debug" "-logFile" "signature.log" \ - "-method" "GET" "-query" "tagging=&versionId=$version_id" "-tagKey" "key" "-tagValue" "value" "-contentMD5"; then + "-method" "GET" "-query" "tagging=&versionId=$version_id" "-tagKey" "key" "-tagValue" "value" "-contentMD5" "--" "$version_id"; then log 2 "error tagging object" return 1 fi diff --git a/tests/drivers/head_object/head_object_rest.sh b/tests/drivers/head_object/head_object_rest.sh index 34793d27..dc6873ba 100644 --- a/tests/drivers/head_object/head_object_rest.sh +++ b/tests/drivers/head_object/head_object_rest.sh @@ -60,10 +60,14 @@ check_checksum_rest() { return 1 fi header_key="$4" - if ! head_object_rest_expect_success_callback "$1" "$2" "CHECKSUM=true" "parse_head_checksum"; then - log 2 "error calling HeadObject command" + + local response + if ! response=$(head_object_rest_expect_success_callback "$1" "$2" "CHECKSUM=true" "parse_head_checksum" 2>&1); then + log 2 "error calling HeadObject command: $response" return 1 fi + + head_checksum="$response" if [ "$3" != "$head_checksum" ]; then log 2 "'checksum mismatch (head '$head_checksum', local '$3')" return 1 @@ -76,6 +80,8 @@ parse_head_checksum() { return 1 fi head_checksum=$(grep -i "$header_key" "$1" | awk '{print $2}' | sed 's/\r$//') + echo "$head_checksum" + return 0 } verify_checksum_doesnt_exist() { @@ -93,7 +99,8 @@ parse_content_length() { if ! check_param_count_v2 "file" 1 $#; then return 1 fi - content_length=$(grep "Content-Length:" "$1" | awk '{print $2}' | tr -d '\r') + content_length="$(grep "Content-Length:" "$1" | awk '{print $2}' | tr -d '\r')" + echo "$content_length" return 0 } @@ -163,10 +170,14 @@ get_object_size_with_user() { if ! check_param_count_v2 "username, password, bucket, key" 4 $#; then return 1 fi - if ! head_object_rest_expect_success_callback "$3" "$4" "AWS_ACCESS_KEY_ID=$1 AWS_SECRET_ACCESS_KEY=$2" "parse_content_length"; then - log 2 "error getting object size" + + local response + if ! response=$(head_object_rest_expect_success_callback "$3" "$4" "AWS_ACCESS_KEY_ID=$1 AWS_SECRET_ACCESS_KEY=$2" "parse_content_length" 2>&1); then + log 2 "error getting object size: $response" return 1 fi + + content_length="$response" log 5 "file size: $content_length" echo "$content_length" return 0 diff --git a/tests/drivers/list_buckets/list_buckets_rest.sh b/tests/drivers/list_buckets/list_buckets_rest.sh index aa46e844..e4cbfa90 100644 --- a/tests/drivers/list_buckets/list_buckets_rest.sh +++ b/tests/drivers/list_buckets/list_buckets_rest.sh @@ -15,10 +15,10 @@ # under the License. bucket_exists_in_list() { - if ! check_param_count_v2 "bucket" 1 $#; then + if ! check_param_count_gt "bucket, buckets in list" 2 $#; then return 1 fi - for bucket in "${bucket_array[@]}"; do + for bucket in "${@:2}"; do if [ "$bucket" == "$1" ]; then return 0 fi @@ -41,18 +41,21 @@ list_check_buckets_rest_with_params() { if ! check_param_count_gt "params, expected buckets" 2 $#; then return 1 fi - if ! list_buckets_rest "$1" "parse_bucket_list"; then - log 2 "error listing buckets" + + local response + if ! response=$(list_buckets_rest "$1" "parse_bucket_list" 2>&1); then + log 2 "error listing buckets: $response" return 1 fi + read -r -a bucket_array <<< "$response" for bucket in "${@:2}"; do log 5 "bucket: $bucket" - if ! bucket_exists_in_list "$bucket"; then - log 2 "bucket $bucket not found" + if ! bucket_exists_in_list "$bucket" "${bucket_array[@]}"; then + log 2 "bucket '$bucket' not found" return 1 fi done - return 0 + return 0 } parse_bucket_list() { @@ -61,7 +64,16 @@ parse_bucket_list() { fi # shellcheck disable=SC2154 log 5 "bucket list: $(cat "$1")" - bucket_list=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$1") + local response + if ! response=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$1" 2>&1); then + if [[ "$response" == *"XPath set is empty"* ]]; then + return 0 + fi + log 2 "error retrieving bucket list: $response" + return 1 + else + bucket_list="$response" + fi bucket_array=() while read -r bucket; do if [ -n "$bucket" ]; then @@ -69,28 +81,50 @@ parse_bucket_list() { bucket_array+=("$bucket") fi done <<< "$bucket_list" - log 5 "bucket array: ${bucket_array[*]}" - log 5 "bucket array length: ${#bucket_array[@]}" + # return val + echo "${bucket_array[*]}" + return 0 } parse_buckets_and_continuation_token() { if ! check_param_count_v2 "data file" 1 $#; then return 1 fi - if ! parse_bucket_list "$1"; then - log 2 "error parsing bucket list" + + local response buckets continuation_token + if ! response=$(parse_bucket_list "$1" 2>&1); then + log 2 "error parsing bucket list: $response" return 1 fi - continuation_token=$(xmllint --xpath '//*[local-name()="ListAllMyBucketsResult"]/*[local-name()="ContinuationToken"]/text()' "$1") - log 5 "token: $continuation_token" + buckets="$response" + + if ! response=$(xmllint --xpath '//*[local-name()="ListAllMyBucketsResult"]/*[local-name()="ContinuationToken"]/text()' "$1" 2>&1); then + if [[ "$response" == *"XPath set is empty"* ]]; then + continuation_token= + else + log 2 "error getting continuation token: $response" + return 1 + fi + else + continuation_token="$response" + fi + + # return vals + echo "$continuation_token" + echo "$buckets" return 0 } check_continuation_token() { - if ! list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token"; then - log 2 "error listing buckets" + local response + if ! response=$(list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token" 2>&1); then + log 2 "error listing buckets: $response" return 1 fi + mapfile -t continuation_token_buckets_lines <<< "$response" + continuation_token="${continuation_token_buckets_lines[0]}" + read -r -a bucket_array <<< "${continuation_token_buckets_lines[1]}" + if [ ${#bucket_array[@]} != "1" ]; then log 2 "expected one bucket to be returned, was ${#bucket_array}" return 1 @@ -105,10 +139,16 @@ check_for_buckets_with_multiple_pages() { if ! check_param_count_v2 "buckets" 2 $#; then return 1 fi - if ! list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token"; then - log 2 "error listing buckets" + + local response + if ! response=$(list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token" 2>&1); then + log 2 "error listing buckets: $response" return 1 fi + mapfile -t continuation_token_buckets_lines <<< "$response" + continuation_token="${continuation_token_buckets_lines[0]}" + read -r -a bucket_array <<< "${continuation_token_buckets_lines[1]}" + bucket_one_found="false" bucket_two_found="false" while true; do @@ -123,10 +163,13 @@ check_for_buckets_with_multiple_pages() { if [ "$continuation_token" == "" ]; then break fi - if ! list_buckets_rest "MAX_BUCKETS=1 CONTINUATION_TOKEN=$continuation_token" "parse_buckets_and_continuation_token"; then - log 2 "error" + if ! response=$(list_buckets_rest "MAX_BUCKETS=1 CONTINUATION_TOKEN=$continuation_token" "parse_buckets_and_continuation_token" 2>&1); then + log 2 "error listing buckets: $response" return 1 fi + mapfile -t continuation_token_buckets_lines <<< "$response" + continuation_token="${continuation_token_buckets_lines[0]}" + read -r -a bucket_array <<< "${continuation_token_buckets_lines[1]}" done if [ "$bucket_one_found" == "false" ]; then log 2 "bucket '$1' not found in list" @@ -313,3 +356,5 @@ list_check_bucket_and_region() { fi return 0 } + + diff --git a/tests/drivers/list_object_versions/list_object_versions_rest.sh b/tests/drivers/list_object_versions/list_object_versions_rest.sh index 89807f78..0ca909a7 100644 --- a/tests/drivers/list_object_versions/list_object_versions_rest.sh +++ b/tests/drivers/list_object_versions/list_object_versions_rest.sh @@ -35,6 +35,7 @@ parse_version_or_delete_marker_id() { return 1 fi log 5 "version or marker ID: $version_or_marker_id" + echo "$version_or_marker_id" return 0 } @@ -42,33 +43,43 @@ parse_version_id() { if ! check_param_count_v2 "data file, IsLatest val" 2 $#; then return 1 fi - if ! parse_version_or_delete_marker_id "$1" "Version" "$2"; then - echo "error parsing version ID" + if ! version_id=$(parse_version_or_delete_marker_id "$1" "Version" "$2" 2>&1); then + echo "error parsing version ID: $version_id" return 1 fi - version_id=$version_or_marker_id + echo "$version_id" return 0 } -parse_non_latest_version_id() { +parse_a_non_latest_version_id() { if ! check_param_count_v2 "data file" 1 $#; then return 1 fi - if ! parse_version_id "$1" "false"; then - log 2 "error getting non-latest version ID" + + local response + if ! response=$(parse_version_id "$1" "false" 2>&1); then + log 2 "error getting non-latest version ID: $response" return 1 fi + + version_id="$response" + echo "$version_id" return 0 } -get_non_latest_version() { +get_a_non_latest_version() { if ! check_param_count_v2 "bucket" $# 1; then return 1 fi - if ! send_rest_go_command_callback "200" "parse_non_latest_version_id" "-method" "GET" "-query" "versions=" "-bucketName" "$1"; then - log 2 "error retrieving version tags" + + local response + if ! response=$(send_rest_go_command_callback "200" "parse_a_non_latest_version_id" "-method" "GET" "-query" "versions=" "-bucketName" "$1" 2>&1); then + log 2 "error retrieving non-latest version ID: $response" return 1 fi + + non_latest_version="$response" + echo "$non_latest_version" return 0 } @@ -76,11 +87,15 @@ check_object_versions_before_deletion() { if ! check_param_count_v2 "data file" 1 $#; then return 1 fi - if ! get_xml_data "$1" "$1.tmp"; then - log 2 "error getting XML data" + + local response + if ! response=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then + log 2 "error getting XML data: $response" return 1 fi - if ! parse_versions_rest "$1.tmp"; then + + xml_data="$response" + if ! parse_versions_rest "$xml_data"; then log 2 "error parsing versions" return 1 fi @@ -90,18 +105,23 @@ check_object_versions_before_deletion() { fi version_id="${version_ids[0]}" log 5 "version ID: $version_id" + echo "$version_id" return 0 } check_object_versions_after_deletion() { - if ! check_param_count_v2 "data file" 1 $#; then + if ! check_param_count_v2 "data file, version ID" 2 $#; then return 1 fi - if ! get_xml_data "$1" "$1.tmp"; then - log 2 "error getting XML data" + + local response + if ! response=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then + log 2 "error getting XML data: $response" return 1 fi - if ! parse_versions_rest "$1.tmp"; then + + xml_data="$response" + if ! parse_versions_rest "$xml_data"; then log 2 "error parsing versions" return 1 fi @@ -109,8 +129,8 @@ check_object_versions_after_deletion() { log 2 "expected version ID count of 2, was '${#version_ids[@]}'" return 1 fi - if [ "${version_ids[0]}" != "$version_id" ]; then - log 2 "expected version ID of '$version_id', was '${version_ids[0]}'" + if [ "${version_ids[0]}" != "$2" ]; then + log 2 "expected version ID of '$2', was '${version_ids[0]}'" return 1 fi if [ "${version_islatests[0]}" != "false" ]; then @@ -128,8 +148,8 @@ list_object_versions_before_and_after_retention_deletion() { if ! check_param_count_v2 "bucket name, file" 2 $#; then return 1 fi - if ! send_rest_go_command_callback "200" "check_object_versions_before_deletion" \ - "-method" "GET" "-bucketName" "$1" "-query" "versions="; then + if ! key_version_id=$(send_rest_go_command_callback "200" "check_object_versions_before_deletion" \ + "-method" "GET" "-bucketName" "$1" "-query" "versions=" 2>&1); then log 2 "error checking versions before deletion" return 1 fi @@ -137,9 +157,10 @@ list_object_versions_before_and_after_retention_deletion() { log 2 "error deleting file" return 1 fi + log 5 "version ID: $key_version_id" if ! send_rest_go_command_callback "200" "check_object_versions_after_deletion" \ - "-method" "GET" "-bucketName" "$1" "-query" "versions="; then - log 2 "error checking versions before deletion" + "-method" "GET" "-bucketName" "$1" "-query" "versions=" "--" "$key_version_id"; then + log 2 "error checking versions after deletion" return 1 fi return 0 @@ -149,10 +170,15 @@ parse_latest_version_id() { if ! check_param_count_v2 "data file" 1 $#; then return 1 fi - if ! parse_version_id "$1" "true"; then - log 2 "error getting latest version ID" + + local response + if ! response=$(parse_version_id "$1" "true" 2>&1); then + log 2 "error getting latest version ID: $response" return 1 fi + + version_id="$response" log 5 "version ID: $version_id" + echo "$version_id" return 0 } diff --git a/tests/drivers/list_objects/list_objects_rest.sh b/tests/drivers/list_objects/list_objects_rest.sh index 2ba9b08f..d3de743c 100644 --- a/tests/drivers/list_objects/list_objects_rest.sh +++ b/tests/drivers/list_objects/list_objects_rest.sh @@ -18,10 +18,13 @@ list_check_single_object() { if ! check_param_count_gt "bucket, key, env params (optional)" 2 $#; then return 1 fi - if ! list_objects_rest "$1" "parse_objects_list_rest" "$3"; then - log 2 "error listing objects" + + local response + if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" "$3" 2>&1); then + log 2 "error listing objects: $response" return 1 fi + mapfile -t object_array <<< "$response" if [ ${#object_array[@]} -ne "1" ]; then log 2 "expected one object, found ${#object_array[@]}" return 1 @@ -154,10 +157,10 @@ check_prefix_delimiter_and_counts() { } list_objects_with_prefix_and_delimiter_check_results() { - if ! check_param_count_gt "bucket name, prefix, delimiter, expected common prefixes, --, expected keys" 5 $#; then + if ! check_param_count_gt "bucket name, ListObjects version, prefix, delimiter, expected common prefixes, --, expected keys" 6 $#; then return 1 fi - if ! send_rest_go_command_callback "200" "check_common_prefixes_and_keys" "-bucketName" "$1" "-query" "delimiter=$3&prefix=$2" "--" "${@:2}"; then + if ! send_rest_go_command_callback "200" "check_common_prefixes_and_keys" "-bucketName" "$1" "-query" "list-type=$2&delimiter=$4&prefix=$3" "--" "${@:3}"; then log 2 "error sending command to list objects or receiving response" return 1 fi @@ -208,6 +211,7 @@ check_count_and_keys() { return 1 fi done + echo "$xml_data" return 0 } @@ -223,3 +227,170 @@ list_objects_check_count_and_keys() { fi return 0 } + +check_count_keys_and_get_token() { + if ! check_param_count_gt "data file, expected continuation token, count, last keys, keys" 4 $#; then + return 1 + fi + + local response error + if ! response=$(check_count_and_keys "$1" "$3" "${@:5}" 2>&1); then + log 2 "error checking count and keys: $response" + return 1 + fi + + xml_file="$response" + if [ "$2" != "" ] && ! error=$(check_if_element_exists "$xml_file" "$2" "ListBucketResult" "ContinuationToken" 2>&1); then + log 2 "error getting continuation token: $error" + return 1 + fi + if [ "$4" == "false" ]; then + if ! response=$(get_element_text "$xml_file" "ListBucketResult" "NextContinuationToken" 2>&1); then + log 2 "error getting next continuation token: $response" + return 1 + fi + next_continuation_token="$response" + else + if response=$(get_element_text "$xml_file" "ListBucketResult" "NextContinuationToken" 2>/dev/null) && [ -n "$response" ]; then + log 2 "last element shouldn't have 'NextConfigurationToken' value" + return 1 + fi + next_continuation_token= + fi + echo "$next_continuation_token" + return 0 +} + +list_objects_v2_check_count_and_keys_get_token() { + if ! check_param_count_gt "bucket name, expected token, count, last keys, keys, additional params if any" 4 $#; then + return 1 + fi + local count="$3" + local keys=("${@:5:$count}") + if ! callback_response=$(send_rest_go_command_callback "200" "check_count_keys_and_get_token" "-bucketName" "$1" "${@:((5+$count))}" "--" "$2" "$count" "$4" "${keys[@]}" 2>&1); then + log 2 "error sending list objects command: $callback_response" + return 1 + else + continuation_token="$callback_response" + fi + echo "$continuation_token" + return 0 +} + +check_start_after_no_continuation_token() { + if ! check_param_count_v2 "bucket, last file in alphabetical order" 2 $#; then + return 1 + fi + if ! send_rest_go_command_callback "200" "verify_element_doesnt_exist" "-bucketName" "$1" "-query" "start-after=$2&list-type=2" "--" "$2" "ListBucketResult" "ContinuationToken"; then + log 2 "error verifying that ContinuationToken value is not returned" + return 1 + fi + return 0 +} + +list_objects_check_start_after_response() { + if ! check_param_count_gt "bucket, start after token, listed files" 2 $#; then + return 1 + fi + if ! send_rest_go_command_callback "200" "check_start_after_response" "-bucketName" "$1" "-query" "start-after=$2&list-type=2" "--" "$2" "${@:3}"; then + log 2 "error listing objects and checking start-after response" + return 1 + fi + return 0 +} + +check_start_after_response() { + if ! check_param_count_gt "data file, start after file, listed files" 2 $#; then + return 1 + fi + + local response + if ! response=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then + log 2 "error parsing xml data: $response" + return 1 + fi + + xml_data="$response" + if ! check_if_element_exists "$xml_data" "$2" "ListBucketResult" "StartAfter"; then + log 2 "error checking if element '$2' exists" + return 1 + fi + local count + if [ "$3" == "" ]; then + count=0 + else + count=$(($#-2)) + fi + if ! check_element_count "$xml_data" "$count" "ListBucketResult" "Contents" "Key"; then + log 2 "error checking element count" + return 1 + fi + for file_name in "${@:3}"; do + if ! check_if_element_exists "$xml_data" "$file_name" "ListBucketResult" "Contents" "Key"; then + log 2 "error checking if element '$file_name' exists" + return 1 + fi + done + return 0 +} + +list_objects_verify_owner_info_missing() { + if ! check_param_count_gt "bucket name, files" 1 $#; then + return 1 + fi + if ! send_rest_go_command_callback "200" "verify_owner_info_missing" "-bucketName" "$1" "-query" "list-type=2&fetch-owner=false" "--" "${@:2}"; then + log 2 "error sending list objects v2 command and verifying that the owner data is missing" + return 1 + fi +} + +verify_owner_info_missing() { + if ! check_param_count_gt "data file, files" 1 $#; then + return 1 + fi + + local response + for key in "${@:2}"; do + if ! response=$(get_element_with_matching_inner_value "$1" "$key" "ListBucketResult" "Contents" "--" "Key" 2>&1); then + log 2 "error finding element matching key '$key': $response" + return 1 + fi + element="$response" + + if check_xml_element_inside_string "$element" "$AWS_ACCESS_KEY_ID" "Owner" "ID"; then + log 2 "'Owner' value should not be present" + return 1 + fi + done + return 0 +} + +list_objects_verify_owner_info_exists() { + if ! check_param_count_gt "bucket name, files" 1 $#; then + return 1 + fi + if ! send_rest_go_command_callback "200" "verify_owner_info_exists" "-bucketName" "$1" "-query" "list-type=2&fetch-owner=true" "--" "${@:2}"; then + log 2 "error sending list objects v2 command and verifying that the owner data is missing" + return 1 + fi +} + +verify_owner_info_exists() { + if ! check_param_count_gt "data file, files" 1 $#; then + return 1 + fi + + local response + for key in "${@:2}"; do + if ! response=$(get_element_with_matching_inner_value "$1" "$key" "ListBucketResult" "Contents" "--" "Key" 2>&1); then + log 2 "error finding element matching key '$key': $response" + else + element="$response" + fi + if ! check_xml_element_inside_string "$element" "$AWS_ACCESS_KEY_ID" "Owner" "ID"; then + log 2 "'Owner' value missing" + return 1 + fi + done + return 0 +} diff --git a/tests/drivers/params.sh b/tests/drivers/params.sh index df2574b8..4d61d0e0 100644 --- a/tests/drivers/params.sh +++ b/tests/drivers/params.sh @@ -73,3 +73,15 @@ check_param_count_le() { fi return 0 } + +check_param_count_ge_le() { + if [ $# -ne 4 ]; then + log 2 "'check_param_count_ge_le' requires params list, expected minimum, expected maximum, actual" + return 1 + fi + if [ "$2" -gt "$4" ] || [ "$3" -lt "$4" ]; then + log_with_stack_ref 2 "function '${FUNCNAME[1]}' requires $1" 2 + return 1 + fi + return 0 +} diff --git a/tests/drivers/put_object_tagging/put_object_tagging_rest.sh b/tests/drivers/put_object_tagging/put_object_tagging_rest.sh index 596287d8..a974f511 100644 --- a/tests/drivers/put_object_tagging/put_object_tagging_rest.sh +++ b/tests/drivers/put_object_tagging/put_object_tagging_rest.sh @@ -44,10 +44,14 @@ tag_old_version() { if ! check_param_count_v2 "bucket name, key" 2 $#; then return 1 fi - if ! get_non_latest_version "$1"; then - log 2 "error getting non-latest object version" + + local response + if ! response=$(get_a_non_latest_version "$1" 2>&1); then + log 2 "error getting non-latest object version: $response" return 1 fi + + version_id="$response" if ! send_rest_go_command "200" "-bucketName" "$1" "-objectKey" "$2" "-query" "versionId=$version_id" "-debug" "-logFile" "signature.log" \ "-commandType" "putObjectTagging" "-tagKey" "key" "-tagValue" "value" "-contentMD5"; then log 2 "error tagging object" diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index fc81d9c8..608427c2 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -36,12 +36,15 @@ check_rest_go_expected_error() { return 1 fi result="$(cat "$1")" - if ! bypass_continues; then - log 2 "error bypassing continues" + + local response + if ! response=$(bypass_continues "$result" 2>&1); then + log 2 "error bypassing continues: $response" return 1 fi + status_code=$(echo -n "$response" | awk '{print $2}') if [ "$2" != "$status_code" ]; then - log 2 "expected curl response '$2', was '$status_code'" + log 2 "expected curl response '$2', was '$status_code' (response: '$result')" return 1 fi if ! check_xml_error_contains "$1" "$3" "$4"; then @@ -99,13 +102,14 @@ check_rest_expected_header_error() { return 1 fi result="$(cat "$1")" - if ! bypass_continues; then - log 2 "error bypassing continues" + + local response + if ! response=$(bypass_continues "$result" 2>&1); then + log 2 "error bypassing continues: $response" return 1 fi - log 5 "status line: $status_line" - status_message=$(echo "$status_line" | cut -d' ' -f3- | tr -d '\r') - log 5 "status code: $status_code, status message: $status_message" + status_code=$(echo "$response" | awk '{print $2}') + status_message=$(echo "$response" | cut -d' ' -f3- | tr -d '\r') if [ "$2" != "$status_code" ]; then log 2 "expected curl response '$2', was '$status_code' ($(echo -n "$result"))" return 1 @@ -171,10 +175,15 @@ send_rest_command_expect_success_callback() { log 2 "expected '$3', was '$response_code' ($(cat "$output_file"))" return 1 fi - if [ "$4" != "" ] && ! "$4" "$output_file"; then - log 2 "callback error" + if [ "$4" == "" ]; then + cat "$output_file" + return 0 + fi + if ! callback_result=$("$4" "$output_file" 2>&1); then + log 2 "callback error: $callback_result" return 1 fi + echo "$callback_result" return 0 } @@ -196,6 +205,7 @@ rest_go_command_perform_send() { return 1 fi echo "$result" + return 0 } send_rest_go_command_expect_error() { @@ -250,11 +260,13 @@ send_rest_go_command_expect_error_callback() { } bypass_continues() { - status_line_idx=1 - status_code="" - continue_count=0 + if ! check_param_count_v2 "raw response" 1 $#; then + return 1 + fi + + local status_line_idx=1 status_code="" continue_count=0 while ((continue_count<10)); do - status_line=$(sed -n "${status_line_idx}p" <<< "$result") + status_line=$(sed -n "${status_line_idx}p" <<< "$1") status_code=$(echo "$status_line" | awk '{print $2}') if [ "$status_code" != "100" ]; then break @@ -266,6 +278,7 @@ bypass_continues() { log 2 "too many continues" return 1 fi + echo "$status_line" return 0 } @@ -311,7 +324,7 @@ send_rest_go_command_callback() { return 1 fi - local all_params=("${@:3}") no_callback_params=0 go_param_array=() callback_params=() + local all_params=("${@:3}") no_callback_params=0 go_param_array=() callback_params=() response if ! params_file=$(get_file_name 2>&1); then log 2 "error getting params file name: $params_file" @@ -324,27 +337,46 @@ send_rest_go_command_callback() { mapfile -t callback_params < <(get_callback_params "${all_params[@]}") fi - if ! rest_go_command_perform_send "${go_param_array[@]}"; then - log 2 "error sending rest go command" + if ! result=$(rest_go_command_perform_send "${go_param_array[@]}" 2>&1); then + log 2 "error sending rest go command: $result" return 1 fi - if ! bypass_continues; then - log 2 "error bypassing continues" + if ! response=$(bypass_continues "$result" 2>&1); then + log 2 "error bypassing continues: $response" return 1 fi + status_code=$(echo -n "$response" | awk '{print $2}') if [ "$1" != "$status_code" ]; then - log 2 "expected curl response '$1', was '$status_code'" + log 2 "expected curl response '$1', was '$status_code' (response: '$result')" return 1 fi - if ! output_file_name=$(get_file_name); then + if [ "$2" == "" ]; then + echo "$result" + return 0 + fi + if ! callback_result=$(call_callback "$result" "$2" 2>&1); then + log 2 "callback error: $callback_result" + return 1 + fi + echo "$callback_result" + return 0 +} + +call_callback() { + if ! check_param_count_v2 "response, callback" 2 $#; then + return 1 + fi + if ! output_file_name=$(get_file_name 2>&1); then log 2 "error generating output file name: $output_file_name" return 1 fi - echo -n "$result" > "$TEST_FILE_FOLDER/$output_file_name" - if [ "$2" != "" ] && ! "$2" "$TEST_FILE_FOLDER/$output_file_name" "${callback_params[@]}"; then - log 2 "error in callback" + echo -n "$1" > "$TEST_FILE_FOLDER/$output_file_name" + local callback_output + if ! callback_output=$("$2" "$TEST_FILE_FOLDER/$output_file_name" "${callback_params[@]}" 2>&1); then + log 2 "error in callback: $callback_output" return 1 fi + echo "$callback_output" return 0 } diff --git a/tests/drivers/upload_part/upload_part_rest.sh b/tests/drivers/upload_part/upload_part_rest.sh index 91683004..521b4917 100644 --- a/tests/drivers/upload_part/upload_part_rest.sh +++ b/tests/drivers/upload_part/upload_part_rest.sh @@ -18,14 +18,22 @@ multipart_upload_rest_before_completion() { if ! check_param_count_v2 "bucket, key, file, part count" 4 $#; then return 1 fi - if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then - log 2 "error creating multipart upload" + + local response + if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then + log 2 "error creating multipart upload: $response" return 1 fi - if ! upload_parts_rest_before_completion "$1" "$2" "$3" "$upload_id" "$4"; then - log 2 "error uploading parts before completion" + upload_id="$response" + + if ! response=$(upload_parts_rest_before_completion "$1" "$2" "$3" "$upload_id" "$4" 2>&1); then + log 2 "error uploading parts before completion: $response" return 1 fi + parts_payload="$response" + + echo "$upload_id $parts_payload" + return 0 } upload_parts_rest_before_completion() { @@ -92,10 +100,14 @@ perform_full_multipart_upload_with_checksum_before_completion() { log 2 "error setting up bucket and large file" return 1 fi - if ! create_multipart_upload_rest "$bucket_name" "$2" "CHECKSUM_TYPE=$3 CHECKSUM_ALGORITHM=$4" "parse_upload_id"; then - log 2 "error creating multipart upload" + + local response + if ! response=$(create_multipart_upload_rest "$bucket_name" "$2" "CHECKSUM_TYPE=$3 CHECKSUM_ALGORITHM=$4" "parse_upload_id" 2>&1); then + log 2 "error creating multipart upload: $response" return 1 fi + upload_id="$response" + lowercase_checksum_algorithm=$(echo -n "$4" | tr '[:upper:]' '[:lower:]') if ! upload_parts_rest_with_checksum_before_completion "$bucket_name" "$2" "$TEST_FILE_FOLDER/$2" "$upload_id" 2 "$lowercase_checksum_algorithm"; then log 2 "error uploading parts" diff --git a/tests/drivers/xml.sh b/tests/drivers/xml.sh index 8794965e..5cc2ebca 100644 --- a/tests/drivers/xml.sh +++ b/tests/drivers/xml.sh @@ -246,7 +246,9 @@ check_if_element_exists() { log 2 "error checking XML data: $data_file" return 1 fi - if ! result=$(xmllint --xpath "boolean(${xpath}[text()='$2'])" "$data_file" 2>&1); then + local search_string="boolean(${xpath}[text()='$2'])" + log 5 "search string: $search_string" + if ! result=$(xmllint --xpath "$search_string" "$data_file" 2>&1); then log 2 "error getting result: $result" return 1 fi @@ -257,6 +259,18 @@ check_if_element_exists() { return 1 } +verify_element_doesnt_exist() { + if ! check_param_count_gt "data file, element, xml tree" 3 $#; then + return 1 + fi + if check_if_element_exists "$1" "$2" "${@:3}"; then + log 2 "element '$2' should not exist in data" + log 5 "data: $(cat "$1")" + return 1 + fi + return 0 +} + print_xml_data_to_file() { if ! check_param_count_v2 "data file" 1 $#; then return 1 diff --git a/tests/rest_scripts/command/canonicalQuery.go b/tests/rest_scripts/command/canonicalQuery.go index 9bd843c1..87639b1e 100644 --- a/tests/rest_scripts/command/canonicalQuery.go +++ b/tests/rest_scripts/command/canonicalQuery.go @@ -34,12 +34,31 @@ func canonicalizeQuery(raw string) (string, error) { raw += "=" } - vals, err := url.ParseQuery(raw) - if err != nil { - return "", fmt.Errorf("error parsing query: %w", err) + // Parse manually instead of url.ParseQuery so we don't treat '+' as space. + // S3 continuation tokens are opaque and may contain literal '+' characters. + pairs := make([]queryPair, 0) + for _, part := range strings.Split(raw, "&") { + if part == "" { + continue + } + kv := strings.SplitN(part, "=", 2) + keyRaw := kv[0] + valRaw := "" + if len(kv) == 2 { + valRaw = kv[1] + } + + key, err := url.PathUnescape(keyRaw) + if err != nil { + return "", fmt.Errorf("error unescaping query key '%s': %w", keyRaw, err) + } + val, err := url.PathUnescape(valRaw) + if err != nil { + return "", fmt.Errorf("error unescaping query value for key '%s': %w", keyRaw, err) + } + pairs = append(pairs, queryPair{key: key, value: val}) } - pairs := getQueryPairs(vals) sort.Slice(pairs, func(i, j int) bool { escapedKeyI, escapedKeyJ := awsQueryEscape(pairs[i].key), awsQueryEscape(pairs[j].key) if escapedKeyI != escapedKeyJ { @@ -60,17 +79,3 @@ func canonicalizeQuery(raw string) (string, error) { } return b.String(), nil } - -func getQueryPairs(values url.Values) []queryPair { - pairs := make([]queryPair, 0, len(values)) - for queryKey, queryValues := range values { - if len(queryValues) == 0 { - pairs = append(pairs, queryPair{key: queryKey, value: ""}) - continue - } - for _, v := range queryValues { - pairs = append(pairs, queryPair{key: queryKey, value: v}) - } - } - return pairs -} diff --git a/tests/test_common.sh b/tests/test_common.sh index b823a03f..a4bf7afb 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -331,12 +331,13 @@ test_common_list_objects_file_count() { run create_test_file_count 1001 assert_success + prefix=$output run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success bucket_name=$output - run put_object_multiple "$1" "$TEST_FILE_FOLDER/file_*" "$bucket_name" + run put_object_multiple "$1" "$TEST_FILE_FOLDER/${prefix}_*" "$bucket_name" assert_success run list_objects_check_file_count "$1" "$bucket_name" 1001 diff --git a/tests/test_rest_list_objects.sh b/tests/test_rest_list_objects.sh index 85e5dcfc..d1db867f 100755 --- a/tests/test_rest_list_objects.sh +++ b/tests/test_rest_list_objects.sh @@ -155,25 +155,7 @@ source ./tests/drivers/list_objects/list_objects_rest.sh } @test "REST - ListObjects - delimiter" { - run get_bucket_name "$BUCKET_ONE_NAME" - assert_success - local bucket_name="$output" - - file_names=("a-b-1.txt" "a-b-2.txt" "a-b/c-1.txt" "a-b/c-2.txt" "a-b/d.txt" "a/c.txt") - local prefix="a-" - run create_test_files_and_folders "${file_names[@]}" - assert_success - - run setup_bucket_v2 "$bucket_name" - assert_success - - for file_name in "${file_names[@]}"; do - run put_object "rest" "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" - assert_success - done - - run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt" - assert_success + list_objects_delimiter "1" } @test "REST - ListObjects - invalid encoding" { @@ -262,3 +244,126 @@ source ./tests/drivers/list_objects/list_objects_rest.sh run list_objects_check_count_and_keys "$bucket_name" "1" "$file_two_name" "-query" "prefix=$file_two_name" assert_success } + +@test "REST - ListObjectsV2 - continuation token" { + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 3 + assert_success + read -r bucket_name file_one file_two file_three <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" "$file_three" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" "$file_three" + assert_success + + run list_objects_v2_check_count_and_keys_get_token "$bucket_name" "" "1" "false" "${sorted_files[0]}" "-query" "list-type=2&max-keys=1" + assert_success + continuation_token=$output + log 5 "continuation token: $continuation_token" + + run list_objects_v2_check_count_and_keys_get_token "$bucket_name" "$continuation_token" "1" "false" "${sorted_files[1]}" "-query" "list-type=2&max-keys=1&continuation-token=$continuation_token" + assert_success + continuation_token=$output + + run list_objects_v2_check_count_and_keys_get_token "$bucket_name" "$continuation_token" "1" "true" "${sorted_files[2]}" "-query" "list-type=2&max-keys=1&continuation-token=$continuation_token" + assert_success +} + +@test "ListObjectsV1 - start-after - error" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2004" + fi + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name file_one file_two <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" + assert_success + + run send_rest_go_command_expect_error "400" "InvalidArgument" "startAfter only supported in REST.GET.BUCKET with list-type=2" \ + "-bucketName" "$bucket_name" "-query" "start-after=$file_one" + assert_success +} + +@test "ListObjectsV1 - start-after - doesn't include continuation token" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2007" + fi + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name file_one file_two <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" + assert_success + + run check_start_after_no_continuation_token "$bucket_name" "$file_two" + assert_success +} + +@test "ListObjectsV2 - start-after - success" { + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name file_one file_two <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" + assert_success + + run list_objects_check_start_after_response "$bucket_name" "${sorted_files[0]}" "${sorted_files[1]}" + assert_success + + run list_objects_check_start_after_response "$bucket_name" "${sorted_files[1]}" + assert_success +} + +@test "ListObjectsV2 - fetch-owner" { + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name file_one file_two <<< "$output" + + mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort) + + run put_objects "$bucket_name" "$file_one" "$file_two" + assert_success + + run list_objects_verify_owner_info_missing "$bucket_name" "$file_one" "$file_two" + assert_success + + run list_objects_verify_owner_info_exists "$bucket_name" "$file_one" "$file_two" + assert_success +} + +# shellcheck disable=SC2030 +@test "ListObjectsV2 - delimiter" { + list_objects_delimiter 2 +} + +list_objects_delimiter() { + run assert_param_count "ListObjects version" 1 $# + assert_success + + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + # shellcheck disable=SC2031 + local bucket_name="$output" + + file_names=("a-b-1.txt" "a-b-2.txt" "a-b/c-1.txt" "a-b/c-2.txt" "a-b/d.txt" "a/c.txt") + local prefix="a-" + run create_test_files_and_folders "${file_names[@]}" + assert_success + + run setup_bucket_v2 "$bucket_name" + assert_success + + for file_name in "${file_names[@]}"; do + run put_object "rest" "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + done + + run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "2" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt" + assert_success +} diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index 355f9d2f..e83db3c1 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -48,17 +48,14 @@ export RUN_USERS=true # complete-multipart-upload @test "test_complete_multipart_upload" { - local bucket_file="bucket-file" - run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=20M count=1 + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 20 + assert_success + read -r bucket_name bucket_file <<< "$output" + + run multipart_upload "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 assert_success - run setup_bucket "$BUCKET_ONE_NAME" - assert_success - - run multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 - assert_success - - run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" + run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" assert_success } @@ -107,35 +104,32 @@ export RUN_USERS=true } @test "test-multipart-upload-from-bucket" { - local bucket_file="bucket-file" - run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=20M count=1 + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 20 + assert_success + read -r bucket_name bucket_file <<< "$output" + + run multipart_upload_from_bucket "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 assert_success - run setup_bucket "$BUCKET_ONE_NAME" - assert_success - - run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 - assert_success - - run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "${bucket_file}-copy" "$TEST_FILE_FOLDER/$bucket_file-copy-two" + run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "${bucket_file}-copy" "$TEST_FILE_FOLDER/$bucket_file-copy-two" assert_success } @test "test_multipart_upload_from_bucket_range_too_large" { - local bucket_file="bucket-file" - run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$bucket_file" + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 20 assert_success + read -r bucket_name bucket_file <<< "$output" - run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" + run multipart_upload_range_too_large "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" assert_success } @test "test_multipart_upload_from_bucket_range_valid" { - local bucket_file="bucket-file" - run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$bucket_file" + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 20 assert_success + read -r bucket_name bucket_file <<< "$output" - run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file" + run run_and_verify_multipart_upload_with_valid_range "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file" assert_success } diff --git a/tests/util/util_list_objects.sh b/tests/util/util_list_objects.sh index 1dcf4563..a7ef081f 100644 --- a/tests/util/util_list_objects.sh +++ b/tests/util/util_list_objects.sh @@ -21,21 +21,23 @@ parse_objects_list_rest() { if ! check_param_count_v2 "data file" 1 $#; then return 1 fi - object_array=() - # shellcheck disable=SC2154 - if ! object_list=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then - if [[ "$object_list" == *"XPath set is empty"* ]]; then + + local response + if ! response=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then + if [[ "$response" == *"XPath set is empty"* ]]; then return 0 fi - log 2 "error getting object list: $object_list" + log 2 "error getting object list: $response" return 1 + else + object_list="$response" fi log 5 "object list: '$object_list'" while IFS= read -r object; do log 5 "parsed key: '$object'" - object_array+=("$(echo -n "$object" | xmlstarlet unesc)") + unescaped_object="$(echo -n "$object" | xmlstarlet unesc)" + echo "$unescaped_object" done <<< "$object_list" - log 5 "object array: ${object_array[*]}" return 0 } @@ -112,7 +114,14 @@ list_check_objects_rest() { if ! check_param_count "list_check_objects_rest" "bucket" 1 $#; then return 1 fi - list_objects_rest "$1" "parse_objects_list_rest" + + local response + if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then + log 2 "error listing and parsing objects: $response" + return 1 + fi + mapfile -t object_array <<< "$response" + object_found=false # shellcheck disable=SC2154 for object in "${object_array[@]}"; do diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh index a064767e..b5f79d8b 100644 --- a/tests/util/util_multipart.sh +++ b/tests/util/util_multipart.sh @@ -62,10 +62,12 @@ multipart_upload_from_bucket() { fi } - if ! create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id"; then - log 2 "error running first multipart upload" + local response + if ! response=$(create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id" 2>&1); then + log 2 "error running first multipart upload: $response" return 1 fi + upload_id="$response" if ! multipart_upload_s3api_complete_from_bucket "$1" "$2" "$4"; then log 2 " error completing multipart upload from bucket" @@ -112,10 +114,13 @@ multipart_upload_from_bucket_range() { fi } - if ! create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id"; then - log 2 "error running first multpart upload" + local response + if ! response=$(create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id" 2>&1); then + log 2 "error running first multpart upload: $response" return 1 fi + upload_id="$response" + parts="[" for ((i = 1; i <= $4; i++)); do if ! upload_part_copy_with_range "$1" "$2-copy" "$upload_id" "$2" "$i" "$5"; then @@ -227,10 +232,14 @@ create_upload_part_copy_rest() { log 2 "error splitting and putting file" return 1 fi - if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then - log 2 "error creating upload and getting ID" + + local response + if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then + log 2 "error creating upload and getting ID: $response" return 1 fi + upload_id="$response" + parts_payload="" for ((i=0; i<=3; i++)); do part_number=$((i+1)) diff --git a/tests/util/util_multipart_abort.sh b/tests/util/util_multipart_abort.sh index 326727d7..525e3dc6 100644 --- a/tests/util/util_multipart_abort.sh +++ b/tests/util/util_multipart_abort.sh @@ -43,10 +43,13 @@ create_abort_multipart_upload_rest() { return 1 fi log 5 "uploads before upload: $(cat "$TEST_FILE_FOLDER/uploads.txt")" - if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then - log 2 "error creating upload" + local response + if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then + log 2 "error creating upload: $response" return 1 fi + upload_id="$response" + if ! list_and_check_upload "$1" "$2" "$upload_id"; then log 2 "error listing multipart uploads after upload creation" return 1 diff --git a/tests/util/util_multipart_before_completion.sh b/tests/util/util_multipart_before_completion.sh index 536aa16b..d4a08e6d 100644 --- a/tests/util/util_multipart_before_completion.sh +++ b/tests/util/util_multipart_before_completion.sh @@ -351,10 +351,11 @@ multipart_upload_range_too_large() { } list_and_check_upload() { - if [ $# -lt 2 ]; then + if ! check_param_count_ge_le "bucket, key, upload ID" 2 3 $#; then log 2 "'list_and_check_upload' requires bucket, key, upload ID (optional)" return 1 fi + if ! uploads=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/uploads.txt" ./tests/rest_scripts/list_multipart_uploads.sh); then log 2 "error listing multipart uploads before upload: $result" return 1 @@ -440,10 +441,14 @@ upload_part_check_etag_header() { if ! check_param_count_v2 "bucket, key, upload ID, part number, part" 5 $#; then return 1 fi - if ! etag=$(upload_part_rest "$1" "$2" "$3" "$4" "$5" 2>&1); then - log 2 "error getting etag: $etag" + + local response + if ! response=$(upload_part_rest "$1" "$2" "$3" "$4" "$5" 2>&1); then + log 2 "error getting etag: $response" return 1 fi + etag="$response" + if ! [[ "$etag" =~ ^\"[0-9a-f]+\" ]]; then log 2 "etag pattern mismatch, etag ($etag) should be hex string surrounded by quotes" return 1 @@ -456,19 +461,31 @@ upload_part_copy_check_etag_header() { log 2 "'upload_part_copy_check_etag_header' requires bucket, destination file, part location" return 1 fi - if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then - log 2 "error creating upload and getting ID: $upload_id" + + local response + if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then + log 2 "error creating upload and getting ID: $response" return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="1" UPLOAD_ID="$upload_id" PART_LOCATION="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part_copy.sh); then + upload_id="$response" + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="1" UPLOAD_ID="$upload_id" PART_LOCATION="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/upload_part_copy.sh); then # shellcheck disable=SC2154 log 2 "error uploading part: $result" return 1 fi - if ! etag=$(get_element_text "$TEST_FILE_FOLDER/response.txt" "CopyPartResult" "ETag"); then - log 2 "error getting etag" + if ! response=$(get_element_text "$TEST_FILE_FOLDER/$file_name" "CopyPartResult" "ETag" 2>&1); then + log 2 "error getting etag: $response" return 1 fi + etag="$response" + log 5 "etag: $etag" if ! [[ "$etag" =~ ^\"[0-9a-f]+\" ]]; then log 2 "etag pattern mismatch, etag ($etag) should be hex string surrounded by quotes" diff --git a/tests/util/util_versioning.sh b/tests/util/util_versioning.sh index 421b1d39..b4750648 100644 --- a/tests/util/util_versioning.sh +++ b/tests/util/util_versioning.sh @@ -83,15 +83,22 @@ echo_versions() { if ! check_param_count_v2 "'Version' or 'DeleteMarker', 'Key' or 'VersionId' or 'IsLatest', file" 3 $#; then return 1 fi - if ! keys=$(xmllint --xpath "//*[local-name()=\"$1\"]/*[local-name()=\"$2\"]/text()" "$3" | xmlstarlet unesc 2>&1); then - if [[ "$keys" == *"XPath set is empty"* ]]; then + + local response + if ! response=$(xmllint --xpath "//*[local-name()=\"$1\"]/*[local-name()=\"$2\"]/text()" "$3" 2>&1); then + if [[ "$response" == *"XPath set is empty"* ]]; then return 0 fi - log 2 "error getting Version 'Key' values: $keys" + log 2 "error getting Version '$1', '$2' values: $response" return 1 + else + keys="$response" fi - log 5 "keys to append: ${keys[*]}" - echo "${keys[*]}" + + unescaped_keys=$(echo -n "$keys" | xmlstarlet unesc) + log 5 "keys to append: ${unescaped_keys[*]}" + echo "${unescaped_keys[*]}" + return 0 } get_base64_version_keys_and_ids() { @@ -131,42 +138,50 @@ parse_versions_rest() { version_keys=() version_ids=() version_islatests=() - if ! keys=$(echo_versions "Version" "Key" "$1"); then - log 2 "error getting Version Key values: $keys" + + local response + if ! response=$(echo_versions "Version" "Key" "$1" 2>&1); then + log 2 "error getting Version Key values: $response" return 1 fi # shellcheck disable=SC2206 - version_keys+=($keys) - if ! ids=$(echo_versions "Version" "VersionId" "$1"); then - log 2 "error getting Version VersionId values: $ids" + version_keys+=($response) + + if ! response=$(echo_versions "Version" "VersionId" "$1" 2>&1); then + log 2 "error getting Version VersionId values: $response" return 1 fi # shellcheck disable=SC2206 - version_ids+=($ids) - if ! is_latest=$(echo_versions "Version" "IsLatest" "$1"); then - log 2 "error getting Version IsLatest values: $is_latest" + version_ids+=($response) + + if ! response=$(echo_versions "Version" "IsLatest" "$1" 2>&1); then + log 2 "error getting Version IsLatest values: $response" return 1 fi # shellcheck disable=SC2206 - version_islatests+=($is_latest) - if ! keys=$(echo_versions "DeleteMarker" "Key" "$1"); then - log 2 "error getting DeleteMarker Key values: $keys" + version_islatests+=($response) + + if ! response=$(echo_versions "DeleteMarker" "Key" "$1" 2>&1); then + log 2 "error getting DeleteMarker Key values: $response" return 1 fi # shellcheck disable=SC2206 - version_keys+=($keys) - if ! ids=$(echo_versions "DeleteMarker" "VersionId" "$1"); then - log 2 "error getting DeleteMarker VersionId values: $ids" + version_keys+=($response) + + if ! response=$(echo_versions "DeleteMarker" "VersionId" "$1" 2>&1); then + log 2 "error getting DeleteMarker VersionId values: $response" return 1 fi # shellcheck disable=SC2206 - version_ids+=($ids) - if ! is_latest=$(echo_versions "DeleteMarker" "IsLatest" "$1"); then - log 2 "error getting DeleteMarker IsLatest values: $is_latest" + version_ids+=($response) + + if ! response=$(echo_versions "DeleteMarker" "IsLatest" "$1" 2>&1); then + log 2 "error getting DeleteMarker IsLatest values: $response" return 1 fi # shellcheck disable=SC2206 - version_islatests+=($is_latest) + version_islatests+=($response) + log 5 "version keys: ${version_keys[*]}" log 5 "version IDs: ${version_ids[*]}" log 5 "base64 pairs: ${base64_pairs[*]}"