From fd8e4cd69cf460558ddbd16b0bcf00c373f7f523 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Mon, 11 May 2026 16:08:43 -0300 Subject: [PATCH] test: HeadObject query tests, more util code changes --- tests/commands/list_buckets.sh | 5 + tests/drivers/head_object/head_object_rest.sh | 18 +++ .../list_buckets/list_buckets.sh} | 37 +----- .../list_buckets/list_buckets_s3api.sh | 42 +++++++ tests/drivers/put_object/put_object_rest.sh | 29 +++++ tests/drivers/rest.sh | 1 + tests/drivers/string.sh | 53 ++++++++ tests/test_common.sh | 1 - tests/test_rest_bucket.sh | 1 - tests/test_rest_head_object.sh | 118 ++++++++++++++---- tests/test_s3.sh | 1 + tests/test_s3api_bucket.sh | 1 + tests/test_s3cmd.sh | 1 + tests/test_user_aws.sh | 2 + tests/test_user_common.sh | 1 - tests/test_user_s3cmd.sh | 2 + 16 files changed, 251 insertions(+), 62 deletions(-) rename tests/{util/util_list_buckets.sh => drivers/list_buckets/list_buckets.sh} (65%) create mode 100644 tests/drivers/list_buckets/list_buckets_s3api.sh create mode 100644 tests/drivers/string.sh diff --git a/tests/commands/list_buckets.sh b/tests/commands/list_buckets.sh index 2157d38a..b4cab7c7 100644 --- a/tests/commands/list_buckets.sh +++ b/tests/commands/list_buckets.sh @@ -50,6 +50,8 @@ list_buckets() { bucket_name=$(echo "$line" | awk '{print $NF}') bucket_array+=("${bucket_name%/}") done <<< "$buckets" + + echo "${bucket_array[*]}" return 0 } @@ -86,6 +88,8 @@ list_buckets_with_user() { bucket_name=$(echo "$line" | awk '{print $NF}') bucket_array+=("${bucket_name%/}") done <<< "$buckets" + + echo "${bucket_array[*]}" return 0 } @@ -111,6 +115,7 @@ list_buckets_s3api() { names=$(jq -r '.Buckets[].Name' <<<"$modified_output") IFS=$'\n' read -rd '' -a bucket_array <<<"$names" + echo "${bucket_array[*]}" return 0 } diff --git a/tests/drivers/head_object/head_object_rest.sh b/tests/drivers/head_object/head_object_rest.sh index e027ab8c..566a57bb 100644 --- a/tests/drivers/head_object/head_object_rest.sh +++ b/tests/drivers/head_object/head_object_rest.sh @@ -236,3 +236,21 @@ head_object_check_header_key_and_value() { fi return 0 } + +check_header_partial_content_response() { + if ! check_param_count_v2 "header data, part number, full object size, part size" 4 $#; then + return 1 + fi + log 5 "header: $(cat "$1")" + starting_byte=$((($2-1)*$4)) + ending_byte=$(($2*$4-1)) + if [ "$3" -lt "$ending_byte" ]; then + ending_byte="$(($3-1))" + fi + content_range_string="bytes $starting_byte-$ending_byte/$3" + if ! result=$(check_for_header_key_and_value "$1" "Content-Range" "$content_range_string" 2>&1); then + log 2 "error checking for header key and value: $result" + return 1 + fi + return 0 +} diff --git a/tests/util/util_list_buckets.sh b/tests/drivers/list_buckets/list_buckets.sh similarity index 65% rename from tests/util/util_list_buckets.sh rename to tests/drivers/list_buckets/list_buckets.sh index eb78a1e0..561b7519 100644 --- a/tests/util/util_list_buckets.sh +++ b/tests/drivers/list_buckets/list_buckets.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright 2024 Versity Software +# Copyright 2026 Versity Software # This file is licensed under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at @@ -19,17 +19,15 @@ list_and_check_buckets_with_user() { log 2 "'list_and_check_buckets' requires client, two bucket names, id, key" return 1 fi - if ! list_buckets_with_user "$1" "$4" "$5"; then - log 2 "error listing buckets" + if ! response=$(list_buckets_with_user "$1" "$4" "$5" 2>&1); then + log 2 "error listing buckets: $response" return 1 fi + read -r -a bucket_array <<< "$response" local bucket_one_found=false local bucket_two_found=false - if [ -z "$bucket_array" ]; then - log 2 "bucket_array parameter not exported" - return 1 - fi + log 5 "bucket array: ${bucket_array[*]}" for bucket in "${bucket_array[@]}"; do if [ "$bucket" == "$2" ] || [ "$bucket" == "s3://$2" ]; then @@ -59,28 +57,3 @@ list_and_check_buckets() { fi return 0 } - -list_and_check_buckets_omit_without_permission() { - if [ $# -ne 4 ]; then - log 2 "'list_and_check_buckets_with_user' requires username, password, non-visible bucket, visible bucket" - return 1 - fi - if ! list_buckets_with_user "s3api" "$1" "$2"; then - log 2 "error listing buckets with user '$1'" - return 1 - fi - bucket_found=false - for bucket in "${bucket_array[@]}"; do - if [ "$bucket" == "$3" ]; then - log 2 "bucket '$3' shouldn't show up in user '$1' bucket list" - return 1 - elif [ "$bucket" == "$4" ]; then - bucket_found=true - fi - done - if [ $bucket_found == false ]; then - log 2 "user-owned bucket '$4' not found in user list" - return 1 - fi - return 0 -} \ No newline at end of file diff --git a/tests/drivers/list_buckets/list_buckets_s3api.sh b/tests/drivers/list_buckets/list_buckets_s3api.sh new file mode 100644 index 00000000..a97f8f1a --- /dev/null +++ b/tests/drivers/list_buckets/list_buckets_s3api.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +list_and_check_buckets_omit_without_permission() { + if [ $# -ne 4 ]; then + log 2 "'list_and_check_buckets_with_user' requires username, password, non-visible bucket, visible bucket" + return 1 + fi + if ! response=$(list_buckets_with_user "s3api" "$1" "$2" 2>&1); then + log 2 "error listing buckets with user '$1': $response" + return 1 + fi + read -r -a bucket_array <<< "$response" + + bucket_found=false + for bucket in "${bucket_array[@]}"; do + if [ "$bucket" == "$3" ]; then + log 2 "bucket '$3' shouldn't show up in user '$1' bucket list" + return 1 + elif [ "$bucket" == "$4" ]; then + bucket_found=true + fi + done + if [ $bucket_found == false ]; then + log 2 "user-owned bucket '$4' not found in user list" + return 1 + fi + return 0 +} diff --git a/tests/drivers/put_object/put_object_rest.sh b/tests/drivers/put_object/put_object_rest.sh index 898fc984..54111a2f 100644 --- a/tests/drivers/put_object/put_object_rest.sh +++ b/tests/drivers/put_object/put_object_rest.sh @@ -71,6 +71,35 @@ setup_bucket_and_add_files() { return 0 } +setup_bucket_and_add_file_v3() { + if ! check_param_count_v2 "bucket name or prefix" 1 $#; then + return 1 + fi + if ! response=$(setup_bucket_v3 "$1" 2>&1); then + log 2 "error setting up bucket: $response" + return 1 + fi + bucket_name="$response" + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! response=$(create_test_files "$file_name" 2>&1); then + log 2 " error creating test file: $response" + return 1 + fi + if ! response=$(put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" 2>&1); then + log 2 "error adding file '$TEST_FILE_FOLDER/$file_name' to bucket '$bucket_name': $response" + return 1 + fi + log 5 "file name: $file_name" + echo "$bucket_name $file_name" + return 0 +} + send_openssl_go_command_chunked_no_content_length() { if ! check_param_count_gt "bucket name, key" 2 $#; then return 1 diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 56138c74..f65d3f1c 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -401,6 +401,7 @@ check_for_header_key_and_value() { if ! check_param_count_v2 "data file, header key, header value" 3 $#; then return 1 fi + log 5 "header data: $(cat "$1")" while IFS=$': \r' read -r key value; do local check_result=0 value="${value%$'\r'}" diff --git a/tests/drivers/string.sh b/tests/drivers/string.sh new file mode 100644 index 00000000..89af58ff --- /dev/null +++ b/tests/drivers/string.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/logger.sh + +generate_random_string() { + if ! check_param_count_v2 "min size, max size" 2 $#; then + return 1 + fi + local string="" range size response + range=$(($2 - $1 + 1)) + size=$(($1 + RANDOM % range)) + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + pipe_error_file="$response" + + while [ "${#string}" -lt "$size" ]; do + chunk="$( + { + dd if=/dev/urandom bs=128 count=1 | + LC_ALL=C tr -d '\000' | + LC_ALL=C tr -dc 'A-Za-z0-9' + } 2>"$TEST_FILE_FOLDER/$pipe_error_file" + )" + status=$? + + if [ $status -ne 0 ]; then + log 2 "error creating chunk: $(cat "$TEST_FILE_FOLDER/$pipe_error_file")" + return 1 + fi + + string="${string}${chunk}" + done + + printf '%s\n' "${string:0:size}" + return 0 +} diff --git a/tests/test_common.sh b/tests/test_common.sh index a4bf7afb..c582e1e9 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -32,7 +32,6 @@ source ./tests/commands/put_public_access_block.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh source ./tests/drivers/file.sh source ./tests/drivers/params.sh -source ./tests/util/util_list_buckets.sh source ./tests/util/util_object.sh source ./tests/util/util_policy.sh source ./tests/util/util_presigned_url.sh diff --git a/tests/test_rest_bucket.sh b/tests/test_rest_bucket.sh index 32563fc7..872395b3 100755 --- a/tests/test_rest_bucket.sh +++ b/tests/test_rest_bucket.sh @@ -29,7 +29,6 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro source ./tests/drivers/put_bucket_tagging/put_bucket_tagging_rest.sh source ./tests/logger.sh source ./tests/setup.sh -source ./tests/util/util_list_buckets.sh source ./tests/util/util_lock_config.sh source ./tests/util/util_public_access_block.sh source ./tests/util/util_rest.sh diff --git a/tests/test_rest_head_object.sh b/tests/test_rest_head_object.sh index e0360c41..83362938 100755 --- a/tests/test_rest_head_object.sh +++ b/tests/test_rest_head_object.sh @@ -19,48 +19,112 @@ load ./bats-assert/load source ./tests/setup.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh source ./tests/drivers/get_object_attributes/get_object_attributes_rest.sh +source ./tests/drivers/string.sh # tags: curl,HeadObject,GetObjectAttributes,ETag,x-amz-object-attributes @test "REST - head object" { - run get_file_name + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" assert_success - test_file="$output" + read -r bucket_name file_name <<< "$output" - run get_bucket_name "$BUCKET_ONE_NAME" - assert_success - bucket_name="$output" - - run setup_bucket_and_file_v2 "$bucket_name" "$test_file" - assert_success - - run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" - assert_success - - run get_etag_rest "$bucket_name" "$test_file" + run get_etag_rest "$bucket_name" "$file_name" assert_success expected_etag=$output - run check_etag_attribute_rest "$bucket_name" "$test_file" "$expected_etag" + run check_etag_attribute_rest "$bucket_name" "$file_name" "$expected_etag" assert_success } # tags: curl,HeadObject,Content-Type @test "REST - HeadObject - default Content-Type is binary/octet-stream" { - run get_bucket_name "$BUCKET_ONE_NAME" + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" assert_success - bucket_name="$output" + read -r bucket_name file_name <<< "$output" - run get_file_name - assert_success - test_file="$output" - - run setup_bucket_and_file_v2 "$bucket_name" "$test_file" - assert_success - - run put_object_rest "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" - assert_success - - run head_object_check_header_key_and_value "$bucket_name" "$test_file" "Content-Type" "binary/octet-stream" + run head_object_check_header_key_and_value "$bucket_name" "$file_name" "Content-Type" "binary/octet-stream" assert_success } + +@test "REST - HeadObject - letter partNumber" { + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run send_rest_go_command "400" "-method" "HEAD" "-query" "partNumber=abc" "-objectKey" "$file_name" "-bucketName" "$bucket_name" + assert_success +} + +@test "REST - HeadObject - invalid partNumber" { + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run send_rest_go_command "416" "-method" "HEAD" "-query" "partNumber=2" "-objectKey" "$file_name" "-bucketName" "$bucket_name" + assert_success +} + +@test "REST - HeadObject - valid partNumbers" { + file_mb=8 + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" "$file_mb" + assert_success + read -r bucket_name test_file <<< "$output" + file_bytes=$((file_mb*1024*1024)) + + run split_file_irregular "$TEST_FILE_FOLDER/$test_file" 5242880 + assert_success + read -r part_one part_two <<< "$output" + log 5 "part one: $part_one, part two: $part_two" + + run perform_multipart_upload_rest_variable_parts "$bucket_name" "$test_file" "$part_one" "$part_two" + assert_success + + run get_file_size "$part_one" + assert_success + part_size_one="$output" + + run get_file_size "$part_one" + assert_success + part_size_two="$output" + + run send_rest_go_command_callback "206" "check_header_partial_content_response" "-bucketName" "$bucket_name" "-objectKey" "$test_file" \ + "-method" "HEAD" "-query" "partNumber=1" "--" "1" "$file_bytes" "$part_size_one" + assert_success + + run send_rest_go_command_callback "206" "check_header_partial_content_response" "-bucketName" "$bucket_name" "-objectKey" "$test_file" \ + "-method" "HEAD" "-query" "partNumber=2" "--" "2" "$file_bytes" "$part_size_two" + assert_success +} + +@test "REST - HeadObject - invalid request header type" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2075" + fi + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run send_rest_go_command "400" "-bucketName" "$bucket_name" "-objectKey" "$file_name" \ + "-method" "HEAD" "-query" "response-invalid=invalid" + assert_success +} + +@test "REST - HeadObject - response queries" { + run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + query_suffixes=("cache-control" "content-disposition" "content-encoding" "content-language" "content-type" "expires") + + for suffix in "${query_suffixes[@]}"; do + run generate_random_string 20 40 + assert_success + value=$output + log 5 "value: $value" + + run send_rest_go_command_callback "200" "check_for_header_key_and_value" "-bucketName" "$bucket_name" "-objectKey" "$file_name" \ + "-method" "HEAD" "-query" "response-$suffix=$value" "--" "$suffix" "$value" + assert_success + done +} diff --git a/tests/test_s3.sh b/tests/test_s3.sh index 468eb886..76374638 100755 --- a/tests/test_s3.sh +++ b/tests/test_s3.sh @@ -18,6 +18,7 @@ load ./bats-support/load load ./bats-assert/load source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/list_buckets/list_buckets.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/drivers/file.sh source ./tests/test_common.sh diff --git a/tests/test_s3api_bucket.sh b/tests/test_s3api_bucket.sh index f1f2f3a0..1d893e77 100755 --- a/tests/test_s3api_bucket.sh +++ b/tests/test_s3api_bucket.sh @@ -52,6 +52,7 @@ source ./tests/drivers/get_bucket_tagging/get_bucket_tagging.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh source ./tests/drivers/head_bucket/head_bucket_rest.sh source ./tests/drivers/head_bucket/head_bucket_s3api.sh +source ./tests/drivers/list_buckets/list_buckets.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/util/util_lock_config.sh source ./tests/util/util_object.sh diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 465a7158..e8a9c070 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -24,6 +24,7 @@ source ./tests/commands/delete_bucket_policy.sh source ./tests/commands/get_bucket_policy.sh source ./tests/commands/put_bucket_policy.sh source ./tests/drivers/get_bucket_location/get_bucket_location.sh +source ./tests/drivers/list_buckets/list_buckets.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/util/util_object.sh diff --git a/tests/test_user_aws.sh b/tests/test_user_aws.sh index 68daed63..a5ce2fd7 100755 --- a/tests/test_user_aws.sh +++ b/tests/test_user_aws.sh @@ -20,6 +20,8 @@ load ./bats-assert/load source ./tests/test_user_common.sh source ./tests/commands/get_object.sh source ./tests/commands/put_object.sh +source ./tests/drivers/list_buckets/list_buckets.sh +source ./tests/drivers/list_buckets/list_buckets_s3api.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/drivers/file.sh source ./tests/drivers/user.sh diff --git a/tests/test_user_common.sh b/tests/test_user_common.sh index 367bdec4..55bacc2a 100755 --- a/tests/test_user_common.sh +++ b/tests/test_user_common.sh @@ -16,7 +16,6 @@ source ./tests/setup.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh -source ./tests/util/util_list_buckets.sh source ./tests/util/util_object.sh source ./tests/util/util_users.sh source ./tests/commands/list_buckets.sh diff --git a/tests/test_user_s3cmd.sh b/tests/test_user_s3cmd.sh index afebe4b2..45257d8f 100755 --- a/tests/test_user_s3cmd.sh +++ b/tests/test_user_s3cmd.sh @@ -17,6 +17,8 @@ load ./bats-support/load load ./bats-assert/load +source ./tests/drivers/list_buckets/list_buckets.sh +source ./tests/drivers/list_buckets/list_buckets_s3api.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/drivers/user.sh source ./tests/test_user_common.sh