From 70c25de544fa3e3a0eac8d11f283f8194b0d9f5a Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Sat, 10 May 2025 20:41:37 -0300 Subject: [PATCH] test: list-buckets tests --- tests/commands/complete_multipart_upload.sh | 20 ++- tests/commands/create_multipart_upload.sh | 36 +---- tests/commands/list_buckets.sh | 33 +--- tests/commands/upload_part.sh | 4 +- .../complete_multipart_upload_rest.sh | 15 ++ .../create_multipart_upload_rest.sh | 27 ++++ .../drivers/list_buckets/list_buckets_rest.sh | 151 ++++++++++++++++++ tests/drivers/rest.sh | 26 +++ tests/drivers/upload_part/upload_part_rest.sh | 18 ++- tests/rest_scripts/README.md | 33 ++++ .../rest_scripts/complete_multipart_upload.sh | 5 + tests/rest_scripts/create_bucket.sh | 2 +- tests/rest_scripts/list_buckets.sh | 50 ++++-- tests/rest_scripts/list_objects.sh | 2 +- tests/test_rest_bucket.sh | 38 ++++- tests/test_rest_multipart.sh | 17 +- tests/util/util_list_buckets.sh | 21 --- tests/util/util_list_parts.sh | 4 +- tests/util/util_multipart.sh | 9 +- tests/util/util_multipart_abort.sh | 2 +- .../util/util_multipart_before_completion.sh | 10 +- tests/util/util_rest.sh | 14 -- 22 files changed, 398 insertions(+), 139 deletions(-) create mode 100644 tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh create mode 100644 tests/drivers/list_buckets/list_buckets_rest.sh create mode 100644 tests/rest_scripts/README.md diff --git a/tests/commands/complete_multipart_upload.sh b/tests/commands/complete_multipart_upload.sh index 45c64b0..3c8b25b 100644 --- a/tests/commands/complete_multipart_upload.sh +++ b/tests/commands/complete_multipart_upload.sh @@ -31,17 +31,27 @@ complete_multipart_upload() { } complete_multipart_upload_rest() { - if ! check_param_count_v2 "bucket, key, upload ID, parts payload" 4 $#; then + if ! check_param_count_gt "bucket, key, upload ID, parts, additional params" 4 $#; then return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" PARTS="$4" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then - log 2 "error completing multipart upload: $result" + env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2 UPLOAD_ID=$3 PARTS=$4 $5" + if ! send_rest_command_expect_success "$env_vars" "./tests/rest_scripts/complete_multipart_upload.sh" "200"; then + log 2 "error sending REST command and checking error" return 1 fi - if [ "$result" != "200" ]; then - log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")" + return 0 +} + +complete_multipart_upload_rest_expect_error() { + if ! check_param_count_gt "bucket, key, upload ID, parts, additional params, response code, error code, message" 8 $#; then return 1 fi + env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2 UPLOAD_ID=$3 PARTS=$4 $5" + if ! send_rest_command_expect_error "$env_vars" "./tests/rest_scripts/complete_multipart_upload.sh" "$6" "$7" "$8"; then + log 2 "error sending REST command and checking error" + return 1 + fi + return 0 } complete_multipart_upload_rest_nonexistent_param() { diff --git a/tests/commands/create_multipart_upload.sh b/tests/commands/create_multipart_upload.sh index 98b922b..6f521af 100644 --- a/tests/commands/create_multipart_upload.sh +++ b/tests/commands/create_multipart_upload.sh @@ -15,42 +15,14 @@ # under the License. create_multipart_upload_rest() { - if ! check_param_count_v2 "bucket name, key" 2 $#; then + if ! check_param_count_v2 "bucket name, key, additional params, callback" 4 $#; then return 1 fi - if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" COMMAND_LOG=$COMMAND_LOG ./tests/rest_scripts/create_multipart_upload.sh); then - log 2 "error creating multipart upload: $result" + env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2 $3" + if ! send_rest_command_expect_success_callback "$env_vars" "./tests/rest_scripts/create_multipart_upload.sh" "200" "$4"; then + log 2 "error sending REST command and checking error" return 1 fi - if [ "$result" != "200" ]; then - log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/output.txt")" - return 1 - fi - if ! upload_id=$(get_element_text "$TEST_FILE_FOLDER/output.txt" "InitiateMultipartUploadResult" "UploadId"); then - log 2 "error getting upload ID: $upload_id" - return 1 - fi - echo "$upload_id" - return 0 -} - -create_multipart_upload_rest_with_checksum_type_and_algorithm() { - if ! check_param_count_v2 "bucket, key, checksum type, checksum algorithm" 4 $#; then - return 1 - fi - if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" CHECKSUM_TYPE="$3" CHECKSUM_ALGORITHM="$4" ./tests/rest_scripts/create_multipart_upload.sh 2>&1); then - log 2 "error creating multipart upload: $result" - return 1 - fi - if [ "$result" != "200" ]; then - log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/output.txt"))" - return 1 - fi - if ! upload_id=$(get_element_text "$TEST_FILE_FOLDER/output.txt" "InitiateMultipartUploadResult" "UploadId"); then - log 2 "error getting upload ID: $upload_id" - return 1 - fi - echo "$upload_id" return 0 } diff --git a/tests/commands/list_buckets.sh b/tests/commands/list_buckets.sh index c4926d8..6fb6a1e 100644 --- a/tests/commands/list_buckets.sh +++ b/tests/commands/list_buckets.sh @@ -32,7 +32,7 @@ list_buckets() { elif [[ $1 == 'mc' ]]; then buckets=$(send_command mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$? elif [[ $1 == 'rest' ]]; then - list_buckets_rest || exit_code=$? + list_buckets_rest "" "parse_bucket_list" || exit_code=$? else log 2 "list buckets command not implemented for '$1'" return 1 @@ -117,36 +117,11 @@ list_buckets_s3api() { } list_buckets_rest() { - if ! result=$(COMMAND_LOG=$COMMAND_LOG OUTPUT_FILE="$TEST_FILE_FOLDER/buckets.txt" ./tests/rest_scripts/list_buckets.sh); then - log 2 "error listing buckets: $result" + if ! check_param_count_v2 "params, callback" 2 $#; then return 1 fi - if [ "$result" != "200" ]; then - log 2 "list-buckets returned code $result: $(cat "$TEST_FILE_FOLDER/buckets.txt")" - return 1 - fi - if ! parse_bucket_list "$TEST_FILE_FOLDER/buckets.txt"; then - log 2 "error parsing bucket list" - return 1 - fi - return 0 -} - -list_buckets_rest_prefix() { - if ! check_param_count_v2 "prefix" 1 $#; then - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" PREFIX="$1" ./tests/rest_scripts/list_buckets.sh 2>&1); then - log 2 "error getting result: $result" - return 1 - fi - if [ "$result" != "200" ]; then - log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" - return 1 - fi - log 5 "buckets w/prefix $1: $(cat "$TEST_FILE_FOLDER/result.txt")" - if ! parse_bucket_list "$TEST_FILE_FOLDER/result.txt"; then - log 2 "error parsing bucket list" + if ! send_rest_command_expect_success_callback "$1" "./tests/rest_scripts/list_buckets.sh" "200" "$2"; then + log 2 "error sending REST command and checking error" return 1 fi return 0 diff --git a/tests/commands/upload_part.sh b/tests/commands/upload_part.sh index 1e24167..04bc0f4 100644 --- a/tests/commands/upload_part.sh +++ b/tests/commands/upload_part.sh @@ -67,7 +67,7 @@ upload_part_rest_without_part_number() { if ! check_param_count_v2 "bucket, key" 2 $#; then return 1 fi - if ! create_multipart_upload_rest "$1" "$2"; then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating multpart upload" return 1 fi @@ -87,7 +87,7 @@ upload_part_rest_without_upload_id() { if ! check_param_count_v2 "bucket, key" 2 $#; then return 1 fi - if ! create_multipart_upload_rest "$1" "$2"; then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating multpart upload" return 1 fi diff --git a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh index 7f2d6ff..dc135ea 100644 --- a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh +++ b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh @@ -165,3 +165,18 @@ test_complete_multipart_upload_invalid_checksum() { fi return 0 } + +complete_multipart_upload_invalid_object_size_string() { + if ! check_param_count_v2 "bucket, key, file" 3 $#; then + return 1 + fi + if ! multipart_upload_rest_before_completion "$1" "$2" "$3" 2; then + log 2 "error performing multipart upload before completion" + return 1 + fi + if ! complete_multipart_upload_rest_expect_error "$1" "$2" "$upload_id" "$parts_payload" "MULTIPART_OBJECT_SIZE=size" "400" "InvalidRequest" "Value for x-amz-mp-object-size header is invalid"; then + log 2 "error completing multipart upload" + return 1 + fi + return 0 +} diff --git a/tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh b/tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh new file mode 100644 index 0000000..35eec02 --- /dev/null +++ b/tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +parse_upload_id() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + if ! upload_id=$(get_element_text "$1" "InitiateMultipartUploadResult" "UploadId"); then + log 2 "error getting upload ID: $upload_id" + return 1 + fi + echo "$upload_id" + return 0 +} \ No newline at end of file diff --git a/tests/drivers/list_buckets/list_buckets_rest.sh b/tests/drivers/list_buckets/list_buckets_rest.sh new file mode 100644 index 0000000..67ff092 --- /dev/null +++ b/tests/drivers/list_buckets/list_buckets_rest.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +bucket_exists_in_list() { + if ! check_param_count_v2 "bucket" 1 $#; then + return 1 + fi + for bucket in "${bucket_array[@]}"; do + if [ "$bucket" == "$1" ]; then + return 0 + fi + done + return 1 +} + +list_check_buckets_rest() { + if ! check_param_count_gt "expected buckets" 1 $#; then + return 1 + fi + if ! list_buckets_rest "" "parse_bucket_list"; then + log 2 "error listing buckets" + return 1 + fi + for bucket in "$@"; do + log 5 "bucket: $bucket" + if ! bucket_exists_in_list "$bucket"; then + log 2 "bucket $bucket not found" + return 1 + fi + done + return 0 +} + +parse_bucket_list() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + # shellcheck disable=SC2154 + log 5 "bucket list: $(cat "$1")" + bucket_list=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$1") + bucket_array=() + while read -r bucket; do + bucket_array+=("$bucket") + done <<< "$bucket_list" + log 5 "bucket array: ${bucket_array[*]}" +} + +parse_buckets_and_continuation_token() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + if ! parse_bucket_list "$1"; then + log 2 "error parsing bucket list" + return 1 + fi + continuation_token=$(xmllint --xpath '//*[local-name()="ListAllMyBucketsResult"]/*[local-name()="ContinuationToken"]/text()' "$1") + log 5 "token: $continuation_token" + return 0 +} + +check_continuation_token() { + if ! list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token"; then + log 2 "error listing buckets" + return 1 + fi + if [ ${#bucket_array[@]} != "1" ]; then + log 2 "expected one bucket to be returned, was ${#bucket_array}" + return 1 + fi + if [ "${bucket_array[0]}" == "$continuation_token" ]; then + log 2 "continuation token shouldn't be bucket name" + return 1 + fi +} + +check_for_buckets_with_multiple_pages() { + if ! check_param_count_v2 "buckets" 2 $#; then + return 1 + fi + if ! list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token"; then + log 2 "error listing buckets" + return 1 + fi + bucket_one_found="false" + bucket_two_found="false" + while true; do + if [ "$bucket_one_found" == "false" ] && [ "${bucket_array[0]}" == "$1" ]; then + bucket_one_found="true" + elif [ "$bucket_two_found" == "false" ] && [ "${bucket_array[0]}" == "$2" ]; then + bucket_two_found="true" + fi + if [ "$bucket_one_found" == "true" ] && [ "$bucket_two_found" == "true" ]; then + break + fi + if [ "$continuation_token" == "" ]; then + break + fi + if ! list_buckets_rest "MAX_BUCKETS=1 CONTINUATION_TOKEN=$continuation_token" "parse_buckets_and_continuation_token"; then + log 2 "error" + return 1 + fi + done + if [ "$bucket_one_found" == "false" ]; then + log 2 "bucket '$1' not found in list" + return 1 + fi + if [ "$bucket_two_found" == "false" ]; then + log 2 "bucket '$2' not found in list" + return 1 + fi + return 0 +} + +list_check_buckets_rest_with_prefix() { + log 6 "list_check_buckets_rest_with_prefix" + if ! check_param_count_v2 "prefix" 1 $#; then + return 1 + fi + if ! list_buckets_rest "PREFIX=$1" "parse_bucket_list"; then + log 2 "error listing buckets with prefix" + return 1 + fi + log 5 "buckets: ${bucket_array[*]}" + local buckets_found=false + for bucket in "$@"; do + log 5 "bucket: $bucket" + if [[ "$bucket" != "$1"* ]]; then + log 2 "bucket doesn't match prefix" + return 1 + fi + buckets_found="true" + done + if [ "$buckets_found" == "false" ]; then + log 2 "no buckets with prefix '$1' found" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 95af823..291d0ca 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -78,3 +78,29 @@ send_rest_command_expect_success() { fi return 0 } + +send_rest_command_expect_success_callback() { + if ! check_param_count_v2 "env vars, script, response code, callback fn" 4 $#; then + return 1 + fi + output_file="$TEST_FILE_FOLDER/output.txt" + local env_array=("env" "COMMAND_LOG=$COMMAND_LOG" "OUTPUT_FILE=$output_file") + if [ "$1" != "" ]; then + IFS=' ' read -r -a env_vars <<< "$1" + env_array+=("${env_vars[@]}") + fi + # shellcheck disable=SC2068 + if ! result=$(${env_array[@]} "$2" 2>&1); then + log 2 "error sending command: $result" + return 1 + fi + if [ "$result" != "$3" ]; then + log 2 "expected '$3', was '$result' ($(cat "$TEST_FILE_FOLDER/output.txt"))" + return 1 + fi + if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/output.txt"; then + log 2 "callback error" + return 1 + fi + return 0 +} diff --git a/tests/drivers/upload_part/upload_part_rest.sh b/tests/drivers/upload_part/upload_part_rest.sh index f3411b4..1815a5f 100644 --- a/tests/drivers/upload_part/upload_part_rest.sh +++ b/tests/drivers/upload_part/upload_part_rest.sh @@ -14,6 +14,20 @@ # specific language governing permissions and limitations # under the License. +multipart_upload_rest_before_completion() { + if ! check_param_count_v2 "bucket, key, file, part count" 4 $#; then + return 1 + fi + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then + log 2 "error creating multipart upload" + return 1 + fi + if ! upload_parts_rest_before_completion "$1" "$2" "$3" "$upload_id" "$4"; then + log 2 "error uploading parts before completion" + return 1 + fi +} + upload_parts_rest_before_completion() { if ! check_param_count_v2 "bucket, key, file, upload ID, part count" 5 $#; then return 1 @@ -22,7 +36,7 @@ upload_parts_rest_before_completion() { log 2 "error splitting file" return 1 fi - local parts_payload="" + parts_payload="" for ((part=0;part<"$5";part++)); do part_number=$((part+1)) if ! etag=$(upload_part_rest "$1" "$2" "$4" "$part_number" "$3-$part" 2>&1); then @@ -68,7 +82,7 @@ perform_full_multipart_upload_with_checksum_before_completion() { log 2 "error setting up bucket and large file" return 1 fi - if ! create_multipart_upload_rest_with_checksum_type_and_algorithm "$1" "$2" "$3" "$4"; then + if ! create_multipart_upload_rest "$1" "$2" "CHECKSUM_TYPE=$3 CHECKSUM_ALGORITHM=$4" "parse_upload_id"; then log 2 "error creating multipart upload" return 1 fi diff --git a/tests/rest_scripts/README.md b/tests/rest_scripts/README.md new file mode 100644 index 0000000..a43f4b6 --- /dev/null +++ b/tests/rest_scripts/README.md @@ -0,0 +1,33 @@ +# REST scripts + +## Parameters + +The common S3 parameters for the scripts can be found in **rest.sh**. They are: +* AWS_ACCESS_KEY_ID +* AWS_SECRET_ACCESS_KEY +* AWS_ENDPOINT_URL +* AWS_REGION + +The `OUTPUT_FILE` parameter can be set to write the response data to a file. + +Operation-specific parameters can be found by looking at the tops of the scripts. For example, for the CreateBucket operation, this would be `BUCKET_NAME`. + +## cURL + +Most scripts will send a cURL message to an s3 server. The scripts all use the following parameters: + +In the root folder, for example, the **create_bucket.sh** script can be run with: + +`AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_ENDPOINT_URL= AWS_REGION= BUCKET_NAME= OUTPUT_FILE= ./tests/rest_scripts/create_bucket.sh` + +A successful bucket creation will return a 200 code on the command line. + +## openssl + +Some scripts will generate a raw REST API message that can be sent via openssl. + +For example, to create this file with **put_object_openssl.sh**: + +`AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_ENDPOINT_URL= AWS_REGION= DATA_FILE= BUCKET_NAME= OBJECT_KEY= COMMAND_FILE= ./tests/rest_scripts/put_object_openssl.sh` + +This should generate a raw REST command file in the `COMMAND_FILE` location. This command can be sent to the S3 server with: `openssl s_client -connect -ign_eof < ` diff --git a/tests/rest_scripts/complete_multipart_upload.sh b/tests/rest_scripts/complete_multipart_upload.sh index ff31668..e9af5c8 100755 --- a/tests/rest_scripts/complete_multipart_upload.sh +++ b/tests/rest_scripts/complete_multipart_upload.sh @@ -34,6 +34,8 @@ checksum_algorithm="$CHECKSUM_ALGORITHM" checksum_hash="$CHECKSUM_HASH" # shellcheck disable=SC2154 algorithm_parameter="${ALGORITHM_PARAMETER:=false}" +# shellcheck disable=SC2153 +multipart_object_size="$MULTIPART_OBJECT_SIZE" payload=" $parts" @@ -53,6 +55,9 @@ if [ "$checksum_type" != "" ]; then cr_data+=("x-amz-checksum-type:$checksum_type") fi cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time") +if [ "$multipart_object_size" != "" ]; then + cr_data+=("x-amz-mp-object-size:$multipart_object_size") +fi build_canonical_request "${cr_data[@]}" # shellcheck disable=SC2119 diff --git a/tests/rest_scripts/create_bucket.sh b/tests/rest_scripts/create_bucket.sh index 2ea94c4..09e85e1 100755 --- a/tests/rest_scripts/create_bucket.sh +++ b/tests/rest_scripts/create_bucket.sh @@ -44,7 +44,7 @@ build_canonical_request "${cr_data[@]}" # shellcheck disable=SC2119 create_canonical_hash_sts_and_signature -curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name") +curl_command+=(curl -ks -w "%{http_code}" -X PUT "$AWS_ENDPOINT_URL/$bucket_name") curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") curl_command+=("${header_fields[@]}") curl_command+=(-o "$OUTPUT_FILE") diff --git a/tests/rest_scripts/list_buckets.sh b/tests/rest_scripts/list_buckets.sh index 7f47fbd..d907336 100755 --- a/tests/rest_scripts/list_buckets.sh +++ b/tests/rest_scripts/list_buckets.sh @@ -16,25 +16,45 @@ source ./tests/rest_scripts/rest.sh +# shellcheck disable=SC2153 +prefix="$PREFIX" +# shellcheck disable=SC2153 +max_buckets="$MAX_BUCKETS" +# shellcheck disable=SC2153 +continuation_token="$CONTINUATION_TOKEN" + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") -canonical_request="GET -/ - -host:$host -x-amz-content-sha256:UNSIGNED-PAYLOAD -x-amz-date:$current_date_time - -host;x-amz-content-sha256;x-amz-date -UNSIGNED-PAYLOAD" +canonical_request_data=("GET" "/") +queries="" +if [ "$continuation_token" != "" ]; then + queries=$(add_parameter "$queries" "continuation-token=$continuation_token") +fi +if [ "$max_buckets" != "" ]; then + queries=$(add_parameter "$queries" "max-buckets=$max_buckets") +fi +if [ "$prefix" != "" ]; then + queries=$(add_parameter "$queries" "prefix=$prefix") +fi +canonical_request_data+=("$queries") +canonical_request_data+=("host:$host" "x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time") +if ! build_canonical_request "${canonical_request_data[@]}"; then + log_rest 2 "error building request" + exit 1 +fi # shellcheck disable=SC2119 create_canonical_hash_sts_and_signature -curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host" --H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" --H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" --H "\"x-amz-date: $current_date_time\"" --o "$OUTPUT_FILE") +curl_command+=(curl -ks -w "\"%{http_code}\"") +url="'$AWS_ENDPOINT_URL" +if [ "$queries" != "" ]; then + url+="?$queries" +fi +url+="'" +curl_command+=("$url") +curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") +curl_command+=("${header_fields[@]}") +curl_command+=(-o "$OUTPUT_FILE") # shellcheck disable=SC2154 -eval "${curl_command[*]}" 2>&1 \ No newline at end of file +eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/list_objects.sh b/tests/rest_scripts/list_objects.sh index b65b48c..158dc32 100755 --- a/tests/rest_scripts/list_objects.sh +++ b/tests/rest_scripts/list_objects.sh @@ -63,7 +63,7 @@ if [ "$queries" != "" ]; then fi url+="'" curl_command+=("$url") -curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"") +curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") curl_command+=("${header_fields[@]}") curl_command+=(-o "$OUTPUT_FILE") # shellcheck disable=SC2154 diff --git a/tests/test_rest_bucket.sh b/tests/test_rest_bucket.sh index c64670d..7bdc72c 100755 --- a/tests/test_rest_bucket.sh +++ b/tests/test_rest_bucket.sh @@ -20,6 +20,7 @@ load ./bats-assert/load source ./tests/commands/get_object_lock_configuration.sh source ./tests/commands/head_bucket.sh source ./tests/commands/list_buckets.sh +source ./tests/drivers/list_buckets/list_buckets_rest.sh source ./tests/logger.sh source ./tests/setup.sh source ./tests/util/util_bucket.sh @@ -68,7 +69,7 @@ export RUN_USERS=true run setup_bucket "$BUCKET_ONE_NAME" assert_success - run list_check_buckets_rest + run list_check_buckets_rest "$BUCKET_ONE_NAME" assert_success } @@ -183,7 +184,7 @@ export RUN_USERS=true run create_bucket_rest "$BUCKET_ONE_NAME" assert_success - run list_check_buckets_rest + run list_check_buckets_rest "$BUCKET_ONE_NAME" assert_success } @@ -280,4 +281,37 @@ export RUN_USERS=true run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/${test_file}-copy" assert_success +} + +@test "REST - list buckets w/prefix" { + run setup_buckets "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" + assert_success + + run list_check_buckets_rest "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" + assert_success + + run list_check_buckets_rest_with_prefix "$BUCKET_ONE_NAME" + assert_success + + run list_check_buckets_rest_with_prefix "$BUCKET_TWO_NAME" + assert_success +} + +@test "REST - list buckets - continuation token isn't bucket name" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1399" + fi + run setup_buckets "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" + assert_success + + run check_continuation_token + assert_success +} + +@test "REST - list buckets - success" { + run setup_buckets "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" + assert_success + + run check_for_buckets_with_multiple_pages "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" + assert_success } \ No newline at end of file diff --git a/tests/test_rest_multipart.sh b/tests/test_rest_multipart.sh index 017251c..e944831 100755 --- a/tests/test_rest_multipart.sh +++ b/tests/test_rest_multipart.sh @@ -114,7 +114,7 @@ test_file="test_file" run split_file "$TEST_FILE_FOLDER/$test_file" 4 assert_success - run create_multipart_upload_rest "$BUCKET_ONE_NAME" "$test_file" + run create_multipart_upload_rest "$BUCKET_ONE_NAME" "$test_file" "" "parse_upload_id" assert_success # shellcheck disable=SC2030 upload_id=$output @@ -212,7 +212,7 @@ test_file="test_file" run setup_bucket "$BUCKET_ONE_NAME" assert_success - run create_multipart_upload_rest_with_checksum_type_and_algorithm "$BUCKET_ONE_NAME" "$test_file" "full_object" "crc64nvme" + run create_multipart_upload_rest "$BUCKET_ONE_NAME" "$test_file" "CHECKSUM_TYPE=full_object CHECKSUM_ALGORITHM=crc64nvme" "parse_upload_id" assert_success } @@ -226,7 +226,7 @@ test_file="test_file" run create_test_file "$test_file" $((5*1024*1024)) assert_success - run create_multipart_upload_rest_with_checksum_type_and_algorithm "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32" + run create_multipart_upload_rest "$BUCKET_ONE_NAME" "$test_file" "CHECKSUM_TYPE=FULL_OBJECT CHECKSUM_ALGORITHM=CRC32" "parse_upload_id" assert_success upload_id=$output log 5 "upload ID: $upload_id" @@ -371,3 +371,14 @@ test_file="test_file" run test_complete_multipart_upload_invalid_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC64NVME" assert_success } + +@test "REST - multipart - x-amz-mp-object-size - invalid string" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1398" + fi + run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run complete_multipart_upload_invalid_object_size_string "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file" + assert_success +} diff --git a/tests/util/util_list_buckets.sh b/tests/util/util_list_buckets.sh index c3107af..eb78a1e 100644 --- a/tests/util/util_list_buckets.sh +++ b/tests/util/util_list_buckets.sh @@ -14,27 +14,6 @@ # specific language governing permissions and limitations # under the License. -list_check_buckets_rest() { - if ! list_buckets "rest"; then - log 2 "error listing buckets" - return 1 - fi - bucket_found=false - # shellcheck disable=SC2154 - for bucket in "${bucket_array[@]}"; do - log 5 "bucket: $bucket" - if [[ $bucket == "$BUCKET_ONE_NAME" ]]; then - bucket_found=true - break - fi - done - if [[ $bucket_found == "false" ]]; then - log 2 "bucket not found" - return 1 - fi - return 0 -} - list_and_check_buckets_with_user() { if [ $# -ne 5 ]; then log 2 "'list_and_check_buckets' requires client, two bucket names, id, key" diff --git a/tests/util/util_list_parts.sh b/tests/util/util_list_parts.sh index 8f22445..eba139f 100644 --- a/tests/util/util_list_parts.sh +++ b/tests/util/util_list_parts.sh @@ -67,7 +67,7 @@ perform_multipart_upload_rest() { log 2 "'upload_check_parts' requires bucket, key, part list" return 1 fi - if ! upload_id=$(create_multipart_upload_rest "$1" "$2" 2>&1); then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating multipart upload" return 1 fi @@ -105,7 +105,7 @@ upload_check_parts() { log 2 "'upload_check_parts' requires bucket, key, part list" return 1 fi - if ! upload_id=$(create_multipart_upload_rest "$1" "$2" 2>&1); then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating upload" return 1 fi diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh index 0e52a48..6412047 100644 --- a/tests/util/util_multipart.sh +++ b/tests/util/util_multipart.sh @@ -15,6 +15,7 @@ # under the License. source ./tests/commands/put_object.sh +source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh multipart_upload_s3api_complete_from_bucket() { if ! check_param_count "multipart_upload_s3api_complete_from_bucket" "bucket, copy source, part count" 3 $#; then @@ -60,7 +61,7 @@ multipart_upload_from_bucket() { fi } - if ! create_multipart_upload_rest "$1" "$2-copy"; then + if ! create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id"; then log 2 "error running first multipart upload" return 1 fi @@ -106,7 +107,7 @@ multipart_upload_from_bucket_range() { fi } - if ! create_multipart_upload_rest "$1" "$2-copy"; then + if ! create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id"; then log 2 "error running first multpart upload" return 1 fi @@ -221,7 +222,7 @@ create_upload_part_copy_rest() { log 2 "error splitting and putting file" return 1 fi - if ! create_multipart_upload_rest "$1" "$2"; then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating upload and getting ID" return 1 fi @@ -258,7 +259,7 @@ create_upload_finish_wrong_etag() { etag="gibberish" part_number=1 - if ! create_multipart_upload_rest "$1" "$2"; then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating upload and getting ID" return 1 fi diff --git a/tests/util/util_multipart_abort.sh b/tests/util/util_multipart_abort.sh index b901c3b..326727d 100644 --- a/tests/util/util_multipart_abort.sh +++ b/tests/util/util_multipart_abort.sh @@ -43,7 +43,7 @@ create_abort_multipart_upload_rest() { return 1 fi log 5 "uploads before upload: $(cat "$TEST_FILE_FOLDER/uploads.txt")" - if ! create_multipart_upload_rest "$1" "$2"; then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating upload" return 1 fi diff --git a/tests/util/util_multipart_before_completion.sh b/tests/util/util_multipart_before_completion.sh index 330e51b..ca37ffc 100644 --- a/tests/util/util_multipart_before_completion.sh +++ b/tests/util/util_multipart_before_completion.sh @@ -200,12 +200,12 @@ create_and_list_multipart_uploads() { return 1 fi - if ! create_multipart_upload_rest "$1" "$2"; then + if ! create_multipart_upload_rest "$1" "$2" "" ""; then log 2 "error creating multpart upload" return 1 fi - if ! create_multipart_upload_rest "$1" "$3"; then + if ! create_multipart_upload_rest "$1" "$3" "" ""; then log 2 "error creating multpart upload two" return 1 fi @@ -455,7 +455,7 @@ upload_part_copy_check_etag_header() { log 2 "'upload_part_copy_check_etag_header' requires bucket, destination file, part location" return 1 fi - if ! upload_id=$(create_multipart_upload_rest "$1" "$2" 2>&1); then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating upload and getting ID: $upload_id" return 1 fi @@ -481,7 +481,7 @@ upload_part_without_part_number() { log 2 "'upload_part_without_upload_id' requires bucket name, key" return 1 fi - if ! create_multipart_upload_rest "$1" "$2"; then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating multpart upload" return 1 fi @@ -502,7 +502,7 @@ upload_part_without_upload_id() { log 2 "'upload_part_without_part_number' requires bucket name, key" return 1 fi - if ! create_multipart_upload_rest "$1" "$2"; then + if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then log 2 "error creating multpart upload" return 1 fi diff --git a/tests/util/util_rest.sh b/tests/util/util_rest.sh index d3d2b18..92c6fbe 100644 --- a/tests/util/util_rest.sh +++ b/tests/util/util_rest.sh @@ -1,19 +1,5 @@ #!/usr/bin/env bash -parse_bucket_list() { - if ! check_param_count_v2 "data file" 1 $#; then - return 1 - fi - # shellcheck disable=SC2154 - log 5 "bucket list: $(cat "$1")" - bucket_list=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$1") - bucket_array=() - while read -r bucket; do - bucket_array+=("$bucket") - done <<< "$bucket_list" - log 5 "bucket array: ${bucket_array[*]}" -} - parse_object_list() { # shellcheck disable=SC2154 object_list=$(echo "$reply" | xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' -)