From 75c25ec789da98d3b072b3f3c64bb0de2fa52699 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Sat, 15 Mar 2025 23:21:43 -0300 Subject: [PATCH] test: chunked upload - more tests, final signature test --- tests/commands/get_bucket_policy.sh | 28 ++++++ .../put_object_openssl_chunked_example.sh | 98 +++++++++++-------- tests/test_rest_chunked.sh | 71 ++++++++++++++ tests/test_s3cmd.sh | 11 +++ tests/util/util_chunked_upload.sh | 77 ++++++++++++++- tests/util/util_file.sh | 22 +++++ tests/util/util_policy.sh | 21 ++-- 7 files changed, 270 insertions(+), 58 deletions(-) diff --git a/tests/commands/get_bucket_policy.sh b/tests/commands/get_bucket_policy.sh index ec12b23..7e61cbd 100644 --- a/tests/commands/get_bucket_policy.sh +++ b/tests/commands/get_bucket_policy.sh @@ -105,6 +105,34 @@ get_bucket_policy_s3cmd() { return 0 } +get_bucket_policy_rest() { + if [[ $# -ne 1 ]]; then + log 2 "s3cmd 'get bucket policy' command requires bucket name" + return 1 + fi + if ! get_bucket_policy_rest_expect_code "$1" "200"; then + log 2 "error getting REST bucket policy" + return 1 + fi + return 0 +} + +get_bucket_policy_rest_expect_code() { + if [[ $# -ne 2 ]]; then + log 2 "s3cmd 'get bucket policy' command requires bucket name, expected code" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then + log 2 "error attempting to get bucket policy response: $result" + return 1 + fi + if [ "$result" != "$2" ]; then + log 2 "unexpected response code, expected '$2', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/policy.txt"))" + return 1 + fi + bucket_policy="$(cat "$TEST_FILE_FOLDER/policy.txt")" +} + # return 0 for no policy, single-line policy, or loading complete, 1 for still searching or loading check_and_load_policy_info() { if [[ $policy_brackets == false ]]; then diff --git a/tests/rest_scripts/put_object_openssl_chunked_example.sh b/tests/rest_scripts/put_object_openssl_chunked_example.sh index 3bfd6d8..7356cb4 100755 --- a/tests/rest_scripts/put_object_openssl_chunked_example.sh +++ b/tests/rest_scripts/put_object_openssl_chunked_example.sh @@ -48,6 +48,8 @@ load_parameters() { # shellcheck disable=SC2153 data_file="$DATA_FILE" chunk_size="${CHUNK_SIZE:=65536}" + # shellcheck disable=SC2153 + final_signature="$FINAL_SIGNATURE" fi readonly initial_sts_data="AWS4-HMAC-SHA256-PAYLOAD @@ -203,8 +205,8 @@ $1 $signature_no_data " if [ "$4" -ne 0 ]; then - if ! data=$(dd if="$2" of="$2.tmp" bs=1 skip="$3" count="$4" 2>&1); then - log_rest 2 "error retrieving data: $data" + if ! error=$(dd if="$2" of="$2.tmp" bs=1 skip="$3" count="$4" 2>&1); then + log_rest 2 "error retrieving data: $error" return 1 fi payload_hash="$(sha256sum "$2.tmp" | awk '{print $1}')" @@ -213,26 +215,31 @@ $signature_no_data fi chunk_sts_data+="$payload_hash" create_canonical_hash_sts_and_signature "$chunk_sts_data" - chunk="$(printf "%x" "$4");chunk-signature=$signature" - if [ "$4" -gt 0 ]; then - chunk+=" -$(cat "$2.tmp")" + if [ -n "$final_signature" ] && [ $((idx+1)) -eq ${#chunk_sizes[@]} ]; then + signature="$final_signature" fi + chunk="$(printf "%x" "$4");chunk-signature=$signature" + echo -e "$chunk\r" >> "$COMMAND_FILE" + if [ "$4" -gt 0 ]; then + dd if="$2.tmp" bs="$4" count=1 >> "$COMMAND_FILE" + echo -e "\r" >> "$COMMAND_FILE" + fi + return 0 } build_chunks() { if [ $# -ne 1 ]; then - log_rest 2 "'add_chunks' requires first signature" + log_rest 2 "'build_chunks' requires first signature" return 1 fi last_signature="$1" idx=0 offset=0 - chunks="" + log_rest 5 "chunk sizes: ${chunk_sizes[*]}" for chunk_size in "${chunk_sizes[@]}"; do if ! build_chunk; then - log 2 "error building chunk" + log_rest 2 "error building chunk" return 1 fi if [ "$test_mode" == "true" ]; then @@ -240,6 +247,7 @@ build_chunks() { fi ((idx++)) done + return 0 } build_chunk() { @@ -256,8 +264,6 @@ build_chunk() { offset=$((offset+chunk_size)) last_signature="$signature" fi - chunks+="$chunk -" } check_chunks_and_signatures_in_test_mode() { @@ -299,62 +305,70 @@ check_chunks_and_signatures_in_test_mode() { esac } -build_command() { - command="PUT /$bucket_name/$key HTTP/1.1 -Host: $host -x-amz-date: $current_date_time -x-amz-storage-class: REDUCED_REDUNDANCY -Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=$first_signature -x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD -Content-Encoding: aws-chunked -x-amz-decoded-content-length: $file_size -Content-Length: $content_length +record_command_lines() { + while IFS= read -r line; do + if ! mask_arg_array "$line"; then + return 1 + fi + # shellcheck disable=SC2154 + echo "${masked_args[*]}" >> "$COMMAND_LOG" + done <<< "$command" +} -" +build_initial_command() { + command="PUT /$bucket_name/$key HTTP/1.1\r +Host: $host\r +x-amz-date: $current_date_time\r +x-amz-storage-class: REDUCED_REDUNDANCY\r +Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=$first_signature\r +x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD\r +Content-Encoding: aws-chunked\r +x-amz-decoded-content-length: $file_size\r +Content-Length: $content_length\r +\r\n" if [ "$test_mode" == "true" ] && [ "$command" != "$expected_command" ]; then log_rest 2 "command mismatch ($command)" return 1 fi + echo -en "$command" > "$COMMAND_FILE" +} - chunks+=" -" - command+="$chunks" - command="${command//$'\n'/$'\r\n'}" - echo -n "$command" > "$COMMAND_FILE" +complete_command() { + echo -e "\r" >> "$COMMAND_FILE" if [ -n "$COMMAND_LOG" ]; then - while IFS= read -r line; do - if ! mask_arg_array "$line"; then - return 1 - fi - # shellcheck disable=SC2154 - echo "${masked_args[*]}" >> "$COMMAND_LOG" - done <<< "$command" + if ! record_command_lines; then + return 1 + fi fi } load_parameters if ! get_file_size_and_content_length; then - log 2 "error getting file size and content length" + log_rest 2 "error getting file size and content length" exit 1 fi if ! get_first_signature; then - log 2 "error getting first signature" + log_rest 2 "error getting first signature" exit 1 fi +if ! build_initial_command; then + log_rest 2 "error building command" + exit 1 +fi if ! build_chunks "$first_signature"; then - log 2 "error building chunks" + log_rest 2 "error building chunks" + exit 1 +fi +if ! complete_command; then + log_rest 2 "error adding chunks" exit 1 fi -if ! build_command; then - log 2 "error building command" - exit 1 -fi if [ "$test_mode" == "true" ]; then - echo "TEST PASS" + log_rest 4 "TEST PASS" fi exit 0 diff --git a/tests/test_rest_chunked.sh b/tests/test_rest_chunked.sh index d647cc9..9fa6f89 100755 --- a/tests/test_rest_chunked.sh +++ b/tests/test_rest_chunked.sh @@ -50,3 +50,74 @@ source ./tests/util/util_setup.sh run attempt_chunked_upload_with_bad_first_signature "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" assert_success } + +@test "REST - chunked upload, final signature error" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1147" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test-file" + run create_test_file "$test_file" 0 + assert_success + + run attempt_chunked_upload_with_bad_final_signature "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "REST - chunked upload, success (file with just a's)" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + sleep 10 + + test_file="test-file" + run create_file_single_char "$test_file" 8192 'a' + assert_success + + run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "REST - chunked upload, success (null bytes)" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + sleep 10 + + test_file="test-file" + run create_file_single_char "$test_file" 8192 '\0' + assert_success + + run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "REST - chunked upload, success (random bytes)" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + sleep 10 + + test_file="test-file" + run create_test_file "$test_file" 10000 + assert_success + + run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "REST - chunked upload, success (zero-byte file)" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + sleep 10 + + test_file="test-file" + run create_test_file "$test_file" 0 + assert_success + + run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success +} diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 87613ab..72406db 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -97,6 +97,17 @@ export RUN_USERS=true test_put_bucket_acl_s3cmd } +@test 's3cmd - can enable public ACLs for bucket' { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1154" + fi + run setup_bucket "s3cmd" "$BUCKET_ONE_NAME" + assert_success + + run put_public_access_block_enable_public_acls "$BUCKET_ONE_NAME" + assert_success +} + # test listing buckets on versitygw @test "test_list_buckets_s3cmd" { test_common_list_buckets "s3cmd" diff --git a/tests/util/util_chunked_upload.sh b/tests/util/util_chunked_upload.sh index 3b7aaab..27cfa49 100644 --- a/tests/util/util_chunked_upload.sh +++ b/tests/util/util_chunked_upload.sh @@ -54,4 +54,79 @@ attempt_chunked_upload_with_bad_first_signature() { return 1 fi return 0 -} \ No newline at end of file +} + +chunked_upload_success() { + if [ $# -ne 3 ]; then + log 2 "'chunked_upload_success_as' requires data file, bucket name, key" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + + host="${AWS_ENDPOINT_URL#http*://}" + if [ "$host" == "s3.amazonaws.com" ]; then + host+=":443" + fi + if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then + log 2 "error sending openssl command: $result" + return 1 + fi + response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')" + if [ "$response_code" != "200" ]; then + log 2 "expected response '200', was '$response_code'" + return 1 + fi + return 0 +} + +attempt_chunked_upload_with_bad_final_signature() { + if [ $# -ne 3 ]; then + log 2 "'attempt_chunked_upload_with_bad_first_signature' requires data file, bucket name, key" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" \ + CHUNK_SIZE=8192 \ + TEST_MODE=false \ + COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" \ + FINAL_SIGNATURE="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + host="${AWS_ENDPOINT_URL#http*://}" + if [ "$host" == "s3.amazonaws.com" ]; then + host+=":443" + fi + if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then + log 2 "error sending openssl command: $result" + return 1 + fi + response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')" + log 5 "response code: $response_code" + if [ "$response_code" != "403" ]; then + log 2 "expected code '403', was '$response_code'" + return 1 + fi + response_data="$(echo "$result" | grep "<")" + log 5 "response data: $response_data" + log 5 "END" + if ! check_xml_element <(echo "$response_data") "SignatureDoesNotMatch" "Error" "Code"; then + log 2 "error checking XML element" + return 1 + fi + return 0 +} diff --git a/tests/util/util_file.sh b/tests/util/util_file.sh index 7d5d653..f996a3b 100644 --- a/tests/util/util_file.sh +++ b/tests/util/util_file.sh @@ -65,6 +65,28 @@ create_test_file() { return 0 } +create_file_single_char() { + if [ "$#" -ne 3 ]; then + log 2 "'create_file_single_char' requires filename, size, char" + return 1 + fi + if [[ -e "$TEST_FILE_FOLDER/$1" ]]; then + if ! error=$(rm "$TEST_FILE_FOLDER/$1" 2>&1); then + log 2 "error removing existing file: $error" + return 1 + fi + fi + if ! error=$(touch "$TEST_FILE_FOLDER/$1" 2>&1); then + log 2 "error creating new file: $error" + return 1 + fi + if ! error=$(dd if=/dev/zero bs=1 count="$2" | tr '\0' "$3" > "$TEST_FILE_FOLDER/$1" 2>&1); then + log 2 "error adding data to file: $error" + return 1 + fi + return 0 +} + # params: folder name # fail if error create_test_folder() { diff --git a/tests/util/util_policy.sh b/tests/util/util_policy.sh index b59b61a..8a5b453 100644 --- a/tests/util/util_policy.sh +++ b/tests/util/util_policy.sh @@ -260,16 +260,12 @@ get_and_check_no_policy_error() { log 2 "'get_and_check_no_policy_error' requires bucket name" return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/get_bucket_policy.sh); then - log 2 "error attempting to get bucket policy response: $result" + if ! get_bucket_policy_rest_expect_code "$1" "404"; then + log 2 "GetBucketPolicy returned unexpected response code" return 1 fi - if [ "$result" != "404" ]; then - log 2 "GetBucketOwnershipControls returned unexpected response code: $result, reply: $(cat "$TEST_FILE_FOLDER/response.txt")" - return 1 - fi - log 5 "response: $(cat "$TEST_FILE_FOLDER/response.txt")" - if ! bucket_name=$(xmllint --xpath '//*[local-name()="BucketName"]/text()' "$TEST_FILE_FOLDER/response.txt" 2>&1); then + log 5 "response: $bucket_policy" + if ! bucket_name=$(xmllint --xpath '//*[local-name()="BucketName"]/text()' <(echo -n "$bucket_policy") 2>&1); then log 2 "error getting bucket name: $bucket_name" return 1 fi @@ -314,21 +310,16 @@ put_and_check_policy_rest() { log 2 "unexpected response code, expected '200' or '204', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/result.txt"))" return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then + if ! get_bucket_policy_rest "$1"; then log 2 "error attempting to get bucket policy response: $result" return 1 fi - if [ "$result" != "200" ]; then - log 2 "unexpected response code, expected '200', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/policy.txt"))" - return 1 - fi - log 5 "policy: $(cat "$TEST_FILE_FOLDER/policy.txt")" if [ "$DIRECT" == "true" ]; then principal="arn:aws:iam::$DIRECT_AWS_USER_ID:user/$4" else principal="$4" fi - if ! check_policy "$(cat "$TEST_FILE_FOLDER/policy.txt")" "$3" "$principal" "$5" "$6"; then + if ! check_policy "$bucket_policy" "$3" "$principal" "$5" "$6"; then log 2 "policies not equal" return 1 fi