From 58659ae2792e879b91e9f0d4b70c5ac203e1226d Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Sat, 10 May 2025 20:41:37 -0300 Subject: [PATCH] test: REST create bucket test, PutObject w/o Content-Length --- tests/commands/delete_object.sh | 24 ++++----- tests/logger.sh | 3 +- tests/rest_scripts/create_bucket.sh | 38 ++++++++++++++ tests/rest_scripts/put_object_openssl.sh | 64 ++++++++++++++++++++++++ tests/test_rest.sh | 26 ++++++++++ tests/test_s3api_object.sh | 2 +- tests/test_s3api_root_inner.sh | 2 +- tests/util/util_chunked_upload.sh | 3 +- tests/util/util_create_bucket.sh | 16 ++++++ tests/util/util_object.sh | 14 ++++++ tests/util/util_retention.sh | 2 +- 11 files changed, 176 insertions(+), 18 deletions(-) create mode 100755 tests/rest_scripts/create_bucket.sh create mode 100755 tests/rest_scripts/put_object_openssl.sh diff --git a/tests/commands/delete_object.sh b/tests/commands/delete_object.sh index bb88bbc..369635f 100644 --- a/tests/commands/delete_object.sh +++ b/tests/commands/delete_object.sh @@ -63,21 +63,19 @@ delete_object_rest() { } delete_object_bypass_retention() { - if ! check_param_count "delete_object_bypass_retention" "client, bucket, key, user, password" 5 $#; then + if ! check_param_count "delete_object_bypass_retention" "bucket, key, user, password" 4 $#; then return 1 fi - if [ "$1" == "rest" ]; then - if ! result=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" \ - COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$2" OBJECT_KEY="$3" BYPASS_GOVERNANCE_RETENTION="true" \ - OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then - log 2 "error deleting object: $result" - return 1 - fi - else - if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1); then - log 2 "error deleting object with bypass retention: $delete_object_error" - return 1 - fi + if ! result=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" \ + COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" BYPASS_GOVERNANCE_RETENTION="true" \ + OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then + log 2 "error deleting object: $result" + return 1 + fi + if [ "$result" != "204" ]; then + delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt") + log 2 "expected '204', was '$result' ($delete_object_error)" + return 1 fi return 0 } diff --git a/tests/logger.sh b/tests/logger.sh index 7c517c7..8f043ed 100644 --- a/tests/logger.sh +++ b/tests/logger.sh @@ -40,7 +40,8 @@ log() { } log_with_stack_ref() { - if ! check_log_params "log_with_stack_ref" "level, string, stack reference" 3 $#; then + if [[ $# -ne 3 ]]; then + echo "log_with_stack_ref function requires level, message, stack reference" return 1 fi # shellcheck disable=SC2153 diff --git a/tests/rest_scripts/create_bucket.sh b/tests/rest_scripts/create_bucket.sh new file mode 100755 index 0000000..359f724 --- /dev/null +++ b/tests/rest_scripts/create_bucket.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +cr_data=("PUT" "/$bucket_name" "" "host:$host") +cr_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time") +build_canonical_request "${cr_data[@]}" + +# shellcheck disable=SC2119 +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name") +curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"") +curl_command+=("${header_fields[@]}") +curl_command+=(-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/put_object_openssl.sh b/tests/rest_scripts/put_object_openssl.sh new file mode 100755 index 0000000..ac16bfb --- /dev/null +++ b/tests/rest_scripts/put_object_openssl.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh +source ./tests/util/util_file.sh + +# Fields + +# shellcheck disable=SC2153 +data_file="$DATA_FILE" +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" +# shellcheck disable=SC2153 +omit_content_length="${OMIT_CONTENT_LENGTH:=false}" +# shellcheck disable=SC2153 +command_file="${COMMAND_FILE:=command.txt}" + +if ! file_size=$(get_file_size "$data_file"); then + log_rest 2 "error getting file size" + exit 1 +fi + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") +payload_hash="$(sha256sum "$data_file" | awk '{print $1}')" + +cr_data=("PUT" "/$bucket_name/$key" "") +cr_data+=("host:$host") +cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time") +build_canonical_request "${cr_data[@]}" + +# shellcheck disable=SC2119 +create_canonical_hash_sts_and_signature + +command="PUT /$bucket_name/$key HTTP/1.1\r +Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\r +" +if [ "$omit_content_length" == "false" ]; then + command+="Content-Length: $file_size +" +fi +for header_field in "${cr_data[@]}"; do + if [[ "$header_field" =~ ^.+:.+$ ]]; then + command+="$header_field\r +" + fi +done +command+="\r\n" +echo -en "$command" > "$command_file" +dd if="$data_file" bs="$file_size" count=1 >> "$command_file" diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 0f0a1ba..07396d3 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -34,6 +34,7 @@ source ./tests/logger.sh source ./tests/setup.sh source ./tests/util/util_acl.sh source ./tests/util/util_attributes.sh +source ./tests/util/util_chunked_upload.sh source ./tests/util/util_delete_object.sh source ./tests/util/util_head_object.sh source ./tests/util/util_legal_hold.sh @@ -654,3 +655,28 @@ test_file="test_file" run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" 2000000 assert_success } + +@test "REST - create bucket test" { + if [ "$RECREATE_BUCKETS" == "false" ]; then + skip "invalid test for static buckets" + fi + run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME" + assert_success + + run create_bucket_rest "$BUCKET_ONE_NAME" + assert_success + + run list_check_buckets_rest + assert_success +} + +@test "REST - put object, missing Content-Length" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1321" + fi + run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run put_object_without_content_length "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file" + assert_success +} diff --git a/tests/test_s3api_object.sh b/tests/test_s3api_object.sh index dd70bb6..826dfa3 100755 --- a/tests/test_s3api_object.sh +++ b/tests/test_s3api_object.sh @@ -277,7 +277,7 @@ export RUN_USERS=true run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" assert_success - run delete_object_bypass_retention "s3api" "$BUCKET_ONE_NAME" "$test_file" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" + run delete_object_bypass_retention "$BUCKET_ONE_NAME" "$test_file" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" assert_failure assert_output -p "InvalidArgument" assert_output -p "x-amz-bypass-governance-retention is only applicable" diff --git a/tests/test_s3api_root_inner.sh b/tests/test_s3api_root_inner.sh index 7a5828e..7898724 100755 --- a/tests/test_s3api_root_inner.sh +++ b/tests/test_s3api_root_inner.sh @@ -208,7 +208,7 @@ test_retention_bypass_s3api_root() { run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" assert_success - run delete_object_bypass_retention "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "$username" "$secret_key" + run delete_object_bypass_retention "$BUCKET_ONE_NAME" "$bucket_file" "$username" "$secret_key" assert_success } diff --git a/tests/util/util_chunked_upload.sh b/tests/util/util_chunked_upload.sh index 3921e37..2e55365 100644 --- a/tests/util/util_chunked_upload.sh +++ b/tests/util/util_chunked_upload.sh @@ -281,9 +281,10 @@ send_via_openssl() { return 1 fi host="${AWS_ENDPOINT_URL#http*://}" - if [ "$host" == "s3.amazonaws.com" ]; then + if [[ "$host" =~ s3\..*amazonaws\.com ]]; then host+=":443" fi + log 5 "connecting to $host" if ! result=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then log 2 "error sending openssl command: $result" return 1 diff --git a/tests/util/util_create_bucket.sh b/tests/util/util_create_bucket.sh index ac82dc3..3b0e480 100644 --- a/tests/util/util_create_bucket.sh +++ b/tests/util/util_create_bucket.sh @@ -59,3 +59,19 @@ create_and_check_bucket_invalid_name() { fi return 0 } + +create_bucket_rest() { + if ! check_param_count "create_bucket_rest" "bucket name" 1 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/create_bucket.sh 2>&1); then + log 2 "error creating bucket: $result" + return 1 + fi + if [ "$result" != "200" ]; then + bucket_create_error="$(cat "$TEST_FILE_FOLDER/result.txt")" + log 2 "expected '200', was '$result' ($bucket_create_error)" + return 1 + fi + return 0 +} diff --git a/tests/util/util_object.sh b/tests/util/util_object.sh index cf7dfaf..e4fa115 100644 --- a/tests/util/util_object.sh +++ b/tests/util/util_object.sh @@ -512,3 +512,17 @@ get_object_with_ranged_download() { file_byte_idx=$((last_byte + 1)) done } + +put_object_without_content_length() { + if ! check_param_count "put_object_without_content_length" "bucket, key, data file" 3 $#; then + return 1 + fi + if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" DATA_FILE="$3" OMIT_CONTENT_LENGTH="true" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl.sh 2>&1); then + log 2 "error getting result: $result" + return 1 + fi + if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 411; then + log 2 "error in sending or checking response code" + return 1 + fi +} diff --git a/tests/util/util_retention.sh b/tests/util/util_retention.sh index 25edc3c..e9c8033 100644 --- a/tests/util/util_retention.sh +++ b/tests/util/util_retention.sh @@ -61,7 +61,7 @@ check_for_and_remove_worm_protection() { if [[ $LOG_LEVEL_INT -ge 5 ]]; then log_worm_protection "$1" "$2" fi - if ! delete_object_bypass_retention "rest" "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then + if ! delete_object_bypass_retention "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then log 2 "error deleting object after legal hold removal" return 2 fi