From 2257281ad1a84e00448a3f048f40cb9ec8c8d132 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Sat, 25 Jan 2025 18:17:12 -0300 Subject: [PATCH] test: chunked uploads w/o trailers - first tests --- .../commands/get_bucket_ownership_controls.sh | 1 + tests/logger.sh | 4 +- tests/rest_scripts/abort_multipart_upload.sh | 1 + .../rest_scripts/complete_multipart_upload.sh | 1 + tests/rest_scripts/copy_object.sh | 1 + tests/rest_scripts/create_multipart_upload.sh | 1 + tests/rest_scripts/delete_object_tagging.sh | 1 + tests/rest_scripts/delete_objects.sh | 1 + tests/rest_scripts/get_bucket_acl.sh | 1 + .../get_bucket_ownership_controls.sh | 1 + tests/rest_scripts/get_bucket_policy.sh | 1 + tests/rest_scripts/get_bucket_tagging.sh | 1 + tests/rest_scripts/get_bucket_versioning.sh | 1 + tests/rest_scripts/get_object.sh | 1 + tests/rest_scripts/get_object_attributes.sh | 1 + tests/rest_scripts/get_object_legal_hold.sh | 1 + tests/rest_scripts/get_public_access_block.sh | 1 + tests/rest_scripts/head_object.sh | 1 + tests/rest_scripts/list_buckets.sh | 1 + tests/rest_scripts/list_multipart_uploads.sh | 1 + tests/rest_scripts/list_object_versions.sh | 1 + tests/rest_scripts/list_objects.sh | 2 + tests/rest_scripts/list_parts.sh | 1 + tests/rest_scripts/put_bucket_acl.sh | 1 + .../put_bucket_ownership_controls.sh | 1 + tests/rest_scripts/put_bucket_policy.sh | 1 + tests/rest_scripts/put_bucket_tagging.sh | 1 + tests/rest_scripts/put_bucket_versioning.sh | 1 + tests/rest_scripts/put_object.sh | 1 + .../put_object_openssl_chunked_example.sh | 351 ++++++++++++++++++ tests/rest_scripts/put_object_retention.sh | 1 + tests/rest_scripts/put_public_access_block.sh | 1 + tests/rest_scripts/rest.sh | 33 +- tests/rest_scripts/upload_part.sh | 1 + tests/rest_scripts/upload_part_copy.sh | 1 + tests/run.sh | 8 +- tests/test_rest_chunked.sh | 55 +++ tests/util/util_acl.sh | 2 + tests/util/util_bucket.sh | 7 +- tests/util/util_chunked_upload.sh | 56 +++ tests/util/util_file.sh | 22 +- tests/util/util_list_parts.sh | 37 +- 42 files changed, 578 insertions(+), 30 deletions(-) create mode 100755 tests/rest_scripts/put_object_openssl_chunked_example.sh create mode 100755 tests/test_rest_chunked.sh create mode 100644 tests/util/util_chunked_upload.sh diff --git a/tests/commands/get_bucket_ownership_controls.sh b/tests/commands/get_bucket_ownership_controls.sh index ff0eb345..d15fafdb 100644 --- a/tests/commands/get_bucket_ownership_controls.sh +++ b/tests/commands/get_bucket_ownership_controls.sh @@ -26,6 +26,7 @@ get_bucket_ownership_controls() { return 1 fi + raw_bucket_ownership_controls="" if ! raw_bucket_ownership_controls=$(send_command aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls" return 1 diff --git a/tests/logger.sh b/tests/logger.sh index ac221382..46be2969 100644 --- a/tests/logger.sh +++ b/tests/logger.sh @@ -16,15 +16,13 @@ # levels: 1 - crit, 2 - err, 3 - warn, 4 - info, 5 - debug, 6 - trace -export LOG_LEVEL_INT=4 - log() { if [[ $# -ne 2 ]]; then echo "log function requires level, message" return 1 fi # shellcheck disable=SC2153 - if [[ $1 -gt $LOG_LEVEL_INT ]]; then + if [[ $1 -gt ${LOG_LEVEL_INT:=4} ]]; then return 0 fi log_level="" diff --git a/tests/rest_scripts/abort_multipart_upload.sh b/tests/rest_scripts/abort_multipart_upload.sh index a3a26acf..bf9924bc 100755 --- a/tests/rest_scripts/abort_multipart_upload.sh +++ b/tests/rest_scripts/abort_multipart_upload.sh @@ -37,6 +37,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id" diff --git a/tests/rest_scripts/complete_multipart_upload.sh b/tests/rest_scripts/complete_multipart_upload.sh index b39a38ed..abf7dc2c 100755 --- a/tests/rest_scripts/complete_multipart_upload.sh +++ b/tests/rest_scripts/complete_multipart_upload.sh @@ -41,6 +41,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id" diff --git a/tests/rest_scripts/copy_object.sh b/tests/rest_scripts/copy_object.sh index 1f72db78..afc5780e 100755 --- a/tests/rest_scripts/copy_object.sh +++ b/tests/rest_scripts/copy_object.sh @@ -38,6 +38,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-copy-source;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key") diff --git a/tests/rest_scripts/create_multipart_upload.sh b/tests/rest_scripts/create_multipart_upload.sh index d9ab4bf6..48b568c8 100755 --- a/tests/rest_scripts/create_multipart_upload.sh +++ b/tests/rest_scripts/create_multipart_upload.sh @@ -39,6 +39,7 @@ UNSIGNED-PAYLOAD" canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploads=" diff --git a/tests/rest_scripts/delete_object_tagging.sh b/tests/rest_scripts/delete_object_tagging.sh index 2cc9f11e..bd7640ca 100755 --- a/tests/rest_scripts/delete_object_tagging.sh +++ b/tests/rest_scripts/delete_object_tagging.sh @@ -35,6 +35,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?tagging" diff --git a/tests/rest_scripts/delete_objects.sh b/tests/rest_scripts/delete_objects.sh index 53550387..79465e2b 100755 --- a/tests/rest_scripts/delete_objects.sh +++ b/tests/rest_scripts/delete_objects.sh @@ -49,6 +49,7 @@ fi canonical_request+="host;x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name?delete") diff --git a/tests/rest_scripts/get_bucket_acl.sh b/tests/rest_scripts/get_bucket_acl.sh index 1b3825c6..314c0fb0 100755 --- a/tests/rest_scripts/get_bucket_acl.sh +++ b/tests/rest_scripts/get_bucket_acl.sh @@ -32,6 +32,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?acl=" diff --git a/tests/rest_scripts/get_bucket_ownership_controls.sh b/tests/rest_scripts/get_bucket_ownership_controls.sh index 4cdf2e62..bc2bf0b9 100755 --- a/tests/rest_scripts/get_bucket_ownership_controls.sh +++ b/tests/rest_scripts/get_bucket_ownership_controls.sh @@ -32,6 +32,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?ownershipControls=" diff --git a/tests/rest_scripts/get_bucket_policy.sh b/tests/rest_scripts/get_bucket_policy.sh index 185627e2..5c33ed95 100755 --- a/tests/rest_scripts/get_bucket_policy.sh +++ b/tests/rest_scripts/get_bucket_policy.sh @@ -32,6 +32,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?policy=" diff --git a/tests/rest_scripts/get_bucket_tagging.sh b/tests/rest_scripts/get_bucket_tagging.sh index 779835a2..919f4b8d 100755 --- a/tests/rest_scripts/get_bucket_tagging.sh +++ b/tests/rest_scripts/get_bucket_tagging.sh @@ -32,6 +32,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?tagging=" diff --git a/tests/rest_scripts/get_bucket_versioning.sh b/tests/rest_scripts/get_bucket_versioning.sh index 10463110..c42eb8cb 100755 --- a/tests/rest_scripts/get_bucket_versioning.sh +++ b/tests/rest_scripts/get_bucket_versioning.sh @@ -32,6 +32,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?versioning=" diff --git a/tests/rest_scripts/get_object.sh b/tests/rest_scripts/get_object.sh index 8ce678ca..b22dc62f 100755 --- a/tests/rest_scripts/get_object.sh +++ b/tests/rest_scripts/get_object.sh @@ -36,6 +36,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key" diff --git a/tests/rest_scripts/get_object_attributes.sh b/tests/rest_scripts/get_object_attributes.sh index b42ef54a..ff9aa5f6 100755 --- a/tests/rest_scripts/get_object_attributes.sh +++ b/tests/rest_scripts/get_object_attributes.sh @@ -41,6 +41,7 @@ x-amz-object-attributes:$attributes host;x-amz-content-sha256;x-amz-date;x-amz-object-attributes UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key?attributes=" diff --git a/tests/rest_scripts/get_object_legal_hold.sh b/tests/rest_scripts/get_object_legal_hold.sh index 93b9cf2d..21c0eaee 100755 --- a/tests/rest_scripts/get_object_legal_hold.sh +++ b/tests/rest_scripts/get_object_legal_hold.sh @@ -35,6 +35,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key?legal-hold=" diff --git a/tests/rest_scripts/get_public_access_block.sh b/tests/rest_scripts/get_public_access_block.sh index 258fb2f9..d9c2871f 100755 --- a/tests/rest_scripts/get_public_access_block.sh +++ b/tests/rest_scripts/get_public_access_block.sh @@ -33,6 +33,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock=" diff --git a/tests/rest_scripts/head_object.sh b/tests/rest_scripts/head_object.sh index 6e0a419b..64be0533 100755 --- a/tests/rest_scripts/head_object.sh +++ b/tests/rest_scripts/head_object.sh @@ -36,6 +36,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ksI -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key" diff --git a/tests/rest_scripts/list_buckets.sh b/tests/rest_scripts/list_buckets.sh index 8ad4cc22..7f47fbd9 100755 --- a/tests/rest_scripts/list_buckets.sh +++ b/tests/rest_scripts/list_buckets.sh @@ -28,6 +28,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host" diff --git a/tests/rest_scripts/list_multipart_uploads.sh b/tests/rest_scripts/list_multipart_uploads.sh index 038b77b9..a68cbf1e 100755 --- a/tests/rest_scripts/list_multipart_uploads.sh +++ b/tests/rest_scripts/list_multipart_uploads.sh @@ -36,6 +36,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name/$key?uploads=" diff --git a/tests/rest_scripts/list_object_versions.sh b/tests/rest_scripts/list_object_versions.sh index 70572c34..89431cf6 100755 --- a/tests/rest_scripts/list_object_versions.sh +++ b/tests/rest_scripts/list_object_versions.sh @@ -34,6 +34,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name?versions" diff --git a/tests/rest_scripts/list_objects.sh b/tests/rest_scripts/list_objects.sh index 55c1415d..f226dba5 100755 --- a/tests/rest_scripts/list_objects.sh +++ b/tests/rest_scripts/list_objects.sh @@ -57,6 +57,8 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" + +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"") diff --git a/tests/rest_scripts/list_parts.sh b/tests/rest_scripts/list_parts.sh index 20eb5560..bb2e1c08 100755 --- a/tests/rest_scripts/list_parts.sh +++ b/tests/rest_scripts/list_parts.sh @@ -38,6 +38,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name/$key?uploadId=$upload_id" diff --git a/tests/rest_scripts/put_bucket_acl.sh b/tests/rest_scripts/put_bucket_acl.sh index 4b53e587..6e8cc02e 100755 --- a/tests/rest_scripts/put_bucket_acl.sh +++ b/tests/rest_scripts/put_bucket_acl.sh @@ -54,6 +54,7 @@ fi canonical_request+="x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl=") diff --git a/tests/rest_scripts/put_bucket_ownership_controls.sh b/tests/rest_scripts/put_bucket_ownership_controls.sh index 220c49cc..095202f5 100755 --- a/tests/rest_scripts/put_bucket_ownership_controls.sh +++ b/tests/rest_scripts/put_bucket_ownership_controls.sh @@ -45,6 +45,7 @@ x-amz-date:$current_date_time content-md5;host;x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?ownershipControls=" diff --git a/tests/rest_scripts/put_bucket_policy.sh b/tests/rest_scripts/put_bucket_policy.sh index b2ab4fd1..300f1207 100755 --- a/tests/rest_scripts/put_bucket_policy.sh +++ b/tests/rest_scripts/put_bucket_policy.sh @@ -38,6 +38,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?policy=" diff --git a/tests/rest_scripts/put_bucket_tagging.sh b/tests/rest_scripts/put_bucket_tagging.sh index 43ab5d59..e930404e 100755 --- a/tests/rest_scripts/put_bucket_tagging.sh +++ b/tests/rest_scripts/put_bucket_tagging.sh @@ -50,6 +50,7 @@ x-amz-date:$current_date_time content-md5;host;x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?tagging=" diff --git a/tests/rest_scripts/put_bucket_versioning.sh b/tests/rest_scripts/put_bucket_versioning.sh index 8432ce5a..d218a1dc 100755 --- a/tests/rest_scripts/put_bucket_versioning.sh +++ b/tests/rest_scripts/put_bucket_versioning.sh @@ -41,6 +41,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?versioning=" diff --git a/tests/rest_scripts/put_object.sh b/tests/rest_scripts/put_object.sh index 2349a3e1..f1cc9633 100755 --- a/tests/rest_scripts/put_object.sh +++ b/tests/rest_scripts/put_object.sh @@ -54,6 +54,7 @@ host;x-amz-content-sha256;x-amz-date $payload_hash" fi +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key") diff --git a/tests/rest_scripts/put_object_openssl_chunked_example.sh b/tests/rest_scripts/put_object_openssl_chunked_example.sh new file mode 100755 index 00000000..c9d3ba77 --- /dev/null +++ b/tests/rest_scripts/put_object_openssl_chunked_example.sh @@ -0,0 +1,351 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh +source ./tests/util/util_file.sh + +# Fields + +load_parameters() { + test_mode=${TEST_MODE:=true} + # shellcheck disable=SC2034 + command_file="${COMMAND_FILE:=command.txt}" + + readonly signature_no_data="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + if [ "$test_mode" == "true" ]; then + current_date_time="20130524T000000Z" + year_month_day="20130524" + bucket_name="examplebucket" + key="chunkObject.txt" + aws_access_key_id="AKIAIOSFODNN7EXAMPLE" + # shellcheck disable=SC2034 + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + aws_region="us-east-1" + host="s3.amazonaws.com" + dd if=/dev/zero bs=1 count=66560 | tr '\0' 'a' > "as.txt" + data_file="as.txt" + chunk_size=65536 + else + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + year_month_day=$(echo "$current_date_time" | cut -c1-8) + # shellcheck disable=SC2153 + bucket_name="$BUCKET_NAME" + key="$OBJECT_KEY" + # shellcheck disable=SC2153 + data_file="$DATA_FILE" + chunk_size="${CHUNK_SIZE:=65536}" + fi + + readonly initial_sts_data="AWS4-HMAC-SHA256-PAYLOAD +$current_date_time +$year_month_day/$aws_region/s3/aws4_request" + + if [ "$test_mode" == "true" ]; then + declare_test_expected_vals + fi +} + +declare_test_expected_vals() { + readonly expected_sts_data="AWS4-HMAC-SHA256 +20130524T000000Z +20130524/us-east-1/s3/aws4_request +cee3fed04b70f867d036f722359b0b1f2f0e5dc0efadbc082b76c4c60e316455" + + readonly expected_sts_chunk_one="AWS4-HMAC-SHA256-PAYLOAD +20130524T000000Z +20130524/us-east-1/s3/aws4_request +4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9 +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +bf718b6f653bebc184e1479f1935b8da974d701b893afcf49e701f3e2f9f9c5a" + + readonly expected_sts_chunk_two="AWS4-HMAC-SHA256-PAYLOAD +20130524T000000Z +20130524/us-east-1/s3/aws4_request +ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648 +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a" + + readonly expected_sts_chunk_three="AWS4-HMAC-SHA256-PAYLOAD +20130524T000000Z +20130524/us-east-1/s3/aws4_request +0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497 +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + readonly expected_canonical_request="PUT +/examplebucket/chunkObject.txt + +content-encoding:aws-chunked +content-length:66824 +host:s3.amazonaws.com +x-amz-content-sha256:STREAMING-AWS4-HMAC-SHA256-PAYLOAD +x-amz-date:20130524T000000Z +x-amz-decoded-content-length:66560 +x-amz-storage-class:REDUCED_REDUNDANCY + +content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class +STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + + readonly expected_command="PUT /examplebucket/chunkObject.txt HTTP/1.1 +Host: s3.amazonaws.com +x-amz-date: 20130524T000000Z +x-amz-storage-class: REDUCED_REDUNDANCY +Authorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9 +x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD +Content-Encoding: aws-chunked +x-amz-decoded-content-length: 66560 +Content-Length: 66824 + +" +} + +get_file_size_and_content_length() { + if ! file_size=$(get_file_size "$data_file"); then + log_rest 2 "error getting file size: $file_size" + return 1 + fi + if [ "$test_mode" == "true" ] && [ "$file_size" != 66560 ]; then + log_rest 2 "file size mismatch ($file_size)" + return 1 + fi + + get_chunk_sizes + content_length=$((length+file_size)) + if [ "$test_mode" == "true" ] && [ "$content_length" != 66824 ]; then + log_rest 2 "content length mismatch ($content_length)" + return 1 + fi +} + +get_chunk_sizes() { + chunk_sizes=() + for ((remaining=file_size; 0&1); then + log_rest 2 "error retrieving data: $data" + return 1 + fi + payload_hash="$(sha256sum "$2.tmp" | awk '{print $1}')" + else + payload_hash="$signature_no_data" + fi + chunk_sts_data+="$payload_hash" + create_canonical_hash_sts_and_signature "$chunk_sts_data" + chunk="$(printf "%x" "$4");chunk-signature=$signature" + if [ "$4" -gt 0 ]; then + chunk+=" +$(cat "$2.tmp")" + fi +} + +build_chunks() { + if [ $# -ne 1 ]; then + log_rest 2 "'add_chunks' requires first signature" + return 1 + fi + + last_signature="$1" + idx=0 + offset=0 + chunks="" + for chunk_size in "${chunk_sizes[@]}"; do + if ! build_chunk; then + log 2 "error building chunk" + return 1 + fi + if [ "$test_mode" == "true" ]; then + check_chunks_and_signatures_in_test_mode $idx + fi + ((idx++)) + done +} + +build_chunk() { + if [ "$chunk_size" == 0 ]; then + if ! create_chunk "$last_signature" "$data_file" 0 0; then + log_rest 2 "error creating chunk $idx" + return 1 + fi + else + if ! create_chunk "$last_signature" "$data_file" "$offset" "${chunk_sizes[$idx]}"; then + log_rest 2 "error creating chunk $idx" + return 1 + fi + offset=$((offset+chunk_size)) + last_signature="$signature" + fi + chunks+="$chunk +" +} + +check_chunks_and_signatures_in_test_mode() { + if [ $# -ne 1 ]; then + log_rest 2 "'check_chunks_and_signatures_in_test_mode' requires chunk number" + return 1 + fi + case "$1" in + 0) + if [ "$chunk_sts_data" != "$expected_sts_chunk_one" ]; then + log_rest 2 "first chunk STS mismatch ($chunk_sts_data)" + return 1 + fi + if [ "$signature" != "ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648" ]; then + log_rest 2 "first chunk signature mismatch ($signature)" + return 1 + fi + ;; + 1) + if [ "$chunk_sts_data" != "$expected_sts_chunk_two" ]; then + log_rest 2 "second chunk STS mismatch ($chunk_sts_data)" + return 1 + fi + if [ "$signature" != "0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497" ]; then + log_rest 2 "second chunk signature mismatch ($signature)" + return 1 + fi + ;; + 2) + if [ "$chunk_sts_data" != "$expected_sts_chunk_three" ]; then + log_rest 2 "final chunk STS mismatch ($chunk_sts_data)" + return 1 + fi + if [ "$signature" != "b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9" ]; then + log_rest 2 "final chunk signature mismatch ($signature)" + return 1 + fi + ;; + esac +} + +build_command() { + command="PUT /$bucket_name/$key HTTP/1.1 +Host: $host +x-amz-date: $current_date_time +x-amz-storage-class: REDUCED_REDUNDANCY +Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=$first_signature +x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD +Content-Encoding: aws-chunked +x-amz-decoded-content-length: $file_size +Content-Length: $content_length + +" + +if [ "$test_mode" == "true" ] && [ "$command" != "$expected_command" ]; then + log_rest 2 "command mismatch ($command)" + return 1 +fi + + chunks+=" +" + command+="$chunks" + command="${command//$'\n'/$'\r\n'}" + echo -n "$command" > "$COMMAND_FILE" +} + +load_parameters + +if ! get_file_size_and_content_length; then + log 2 "error getting file size and content length" + exit 1 +fi + +if ! get_first_signature; then + log 2 "error getting first signature" + exit 1 +fi + +if ! build_chunks "$first_signature"; then + log 2 "error building chunks" + exit 1 +fi + +if ! build_command; then + log 2 "error building command" + exit 1 +fi +if [ "$test_mode" == "true" ]; then + echo "TEST PASS" +fi +exit 0 diff --git a/tests/rest_scripts/put_object_retention.sh b/tests/rest_scripts/put_object_retention.sh index f7135f02..6cf092dc 100755 --- a/tests/rest_scripts/put_object_retention.sh +++ b/tests/rest_scripts/put_object_retention.sh @@ -48,6 +48,7 @@ x-amz-date:$current_date_time content-md5;host;x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key?retention=" diff --git a/tests/rest_scripts/put_public_access_block.sh b/tests/rest_scripts/put_public_access_block.sh index eb0a1233..a1c4aabe 100755 --- a/tests/rest_scripts/put_public_access_block.sh +++ b/tests/rest_scripts/put_public_access_block.sh @@ -46,6 +46,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock=" diff --git a/tests/rest_scripts/rest.sh b/tests/rest_scripts/rest.sh index 3ea3c44d..bb38202b 100644 --- a/tests/rest_scripts/rest.sh +++ b/tests/rest_scripts/rest.sh @@ -15,6 +15,7 @@ # under the License. source ./tests/commands/command.sh +source ./tests/logger.sh # shellcheck disable=SC2153,SC2034 aws_access_key_id="$AWS_ACCESS_KEY_ID" @@ -48,10 +49,14 @@ create_canonical_hash_sts_and_signature() { # shellcheck disable=SC2154 year_month_day="$(echo "$current_date_time" | cut -c1-8)" - sts_data="AWS4-HMAC-SHA256 + if [ $# -eq 0 ]; then + sts_data="AWS4-HMAC-SHA256 $current_date_time $year_month_day/$aws_region/s3/aws4_request $canonical_request_hash" + else + sts_data="$1" + fi date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') @@ -62,18 +67,36 @@ $canonical_request_hash" -mac HMAC \ -macopt hexkey:"$signing_key" | awk '{print $2}') - curl_command=() - add_command_recording_if_enabled + if [ $# -eq 0 ]; then + curl_command=() + add_command_recording_if_enabled + fi } add_parameter() { - if [ "$#" -ne 2 ]; then + if [ "$#" -lt 2 ]; then return fi + if [ "$3" != "" ]; then + divider="$3" + else + divider="&" + fi current_string="$1" if [ "$current_string" != "" ]; then - current_string+="&" + current_string+="$divider" fi current_string+="$2" echo "$current_string" } + +log_rest() { + if [ $# -ne 2 ]; then + return 1 + fi + if [ "$BATS_TEST_NAME" != "" ]; then + log "$1" "$2" + else + echo "$2" + fi +} diff --git a/tests/rest_scripts/upload_part.sh b/tests/rest_scripts/upload_part.sh index eff4c748..6538eee2 100755 --- a/tests/rest_scripts/upload_part.sh +++ b/tests/rest_scripts/upload_part.sh @@ -44,6 +44,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date $payload_hash" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -isk -w "\"%{http_code}\"" "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\"" diff --git a/tests/rest_scripts/upload_part_copy.sh b/tests/rest_scripts/upload_part_copy.sh index e849731a..d317ef3f 100755 --- a/tests/rest_scripts/upload_part_copy.sh +++ b/tests/rest_scripts/upload_part_copy.sh @@ -43,6 +43,7 @@ x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-copy-source;x-amz-date UNSIGNED-PAYLOAD" +# shellcheck disable=SC2119 create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\"" diff --git a/tests/run.sh b/tests/run.sh index 9cb73a6a..b3ebf073 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -141,9 +141,11 @@ run_suite() { ;; rest) echo "Running rest tests ..." - if "$HOME"/bin/bats ./tests/test_rest.sh; then - "$HOME"/bin/bats ./tests/test_rest_acl.sh || exit_code=$? - else + if ! "$HOME"/bin/bats ./tests/test_rest.sh; then + exit_code=1 + elif ! "$HOME"/bin/bats ./tests/test_rest_acl.sh; then + exit_code=1 + elif ! "$HOME"/bin/bats ./tests/test_rest_chunked.sh; then exit_code=1 fi ;; diff --git a/tests/test_rest_chunked.sh b/tests/test_rest_chunked.sh new file mode 100755 index 00000000..0c4b3de5 --- /dev/null +++ b/tests/test_rest_chunked.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/logger.sh +source ./tests/setup.sh +source ./tests/util/util_bucket.sh +source ./tests/util/util_chunked_upload.sh +source ./tests/util/util_file.sh +source ./tests/util/util_head_object.sh + +@test "REST - chunked upload, no content length" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1043" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test-file" + run create_test_files "$test_file" + assert_success + + run attempt_seed_signature_without_content_length "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file" + assert_success +} + +@test "REST - chunked upload, signature error" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1056 - gibberish at end" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test-file" + run create_test_file "$test_file" 8192 + assert_success + + run attempt_chunked_upload_with_bad_first_signature "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success +} diff --git a/tests/util/util_acl.sh b/tests/util/util_acl.sh index 8902ee7f..6e67035d 100644 --- a/tests/util/util_acl.sh +++ b/tests/util/util_acl.sh @@ -401,10 +401,12 @@ check_ownership_rule_and_reset_acl() { return 1 fi # shellcheck disable=SC2154 + log 5 "ownership controls: $bucket_ownership_controls" if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then log 2 "error getting object ownership rule: $object_ownership_rule" return 1 fi + log 5 "ownership rule: $object_ownership_rule" if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then log 2 "error resetting bucket ACL" return 1 diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index 580fb6ca..4ecae20a 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -55,11 +55,8 @@ clear_bucket_s3api() { return 1 fi - if [[ $LOG_LEVEL_INT -ge 5 ]]; then - if ! log_bucket_policy "$1"; then - log 2 "error logging bucket policy" - return 1 - fi + if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$1"; then + log 3 "error logging bucket policy" fi if ! check_object_lock_config "$1"; then diff --git a/tests/util/util_chunked_upload.sh b/tests/util/util_chunked_upload.sh new file mode 100644 index 00000000..933bfc47 --- /dev/null +++ b/tests/util/util_chunked_upload.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +attempt_seed_signature_without_content_length() { + if [ "$#" -ne 3 ]; then + log 2 "'attempt_seed_signature_without_content_length' requires bucket name, key, data file" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" CONTENT_ENCODING="aws-chunked" BUCKET_NAME="$1" OBJECT_KEY="$2" DATA_FILE="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then + log 2 "error putting object: $result" + return 1 + fi + if [ "$result" != 411 ]; then + log 2 "expected '411', actual '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" + return 1 + fi + return 0 +} + +attempt_chunked_upload_with_bad_first_signature() { + if [ $# -ne 3 ]; then + log 2 "'attempt_chunked_upload_with_bad_first_signature' requires data file, bucket name, key" + return 1 + fi + if ! result=$(AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" FIRST_SIGNATURE="xxxxxxxx" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + + host="${AWS_ENDPOINT_URL#http*://}" + if [ "$host" == "s3.amazonaws.com" ]; then + host+=":443" + fi + if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then + log 2 "error sending openssl command: $result" + return 1 + fi + response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')" + log 5 "response code: $response_code" + if [ "$response_code" != "403" ]; then + log 2 "expected code '403', was '$response_code'" + return 1 + fi + response_data="$(echo "$result" | grep "<")" + log 5 "response data: $response_data" + log 5 "END" + if ! check_xml_element <(echo "$response_data") "SignatureDoesNotMatch" "Error" "Code"; then + log 2 "error checking XML element" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/util/util_file.sh b/tests/util/util_file.sh index f53863ac..7d5d653f 100644 --- a/tests/util/util_file.sh +++ b/tests/util/util_file.sh @@ -46,7 +46,7 @@ create_test_file() { return 1 fi fi - if ! error=$(touch "$TEST_FILE_FOLDER/$1"); then + if ! error=$(touch "$TEST_FILE_FOLDER/$1" 2>&1); then log 2 "error creating new file: $error" return 1 fi @@ -102,6 +102,26 @@ delete_test_files() { return 0 } +get_file_size() { + if [ $# -ne 1 ]; then + log 2 "'get_file_size' requires file location" + return 1 + fi + local file_size="" + if [[ "$OSTYPE" == "darwin"* ]]; then + if ! file_size=$(stat -f %z "$1" 2>&1); then + log 2 "error getting file size: $file_size" + return 1 + fi + else + if ! file_size=$(stat -c %s "$1" 2>&1); then + log 2 "error getting file size: $file_size" + return 1 + fi + fi + echo "$file_size" +} + # split file into pieces to test multipart upload # param: file location # return 0 for success, 1 for error diff --git a/tests/util/util_list_parts.sh b/tests/util/util_list_parts.sh index 611bc51c..52d588f7 100644 --- a/tests/util/util_list_parts.sh +++ b/tests/util/util_list_parts.sh @@ -1,5 +1,27 @@ #!/usr/bin/env bash +check_part_list_rest_etags() { + if ! etags=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/parts.txt" | tr '\n' ' ' 2>&1); then + log 2 "error retrieving etags: $etags" + return 1 + fi + read -ra etags_array <<< "$etags" + idx=0 + if [ $# -ne ${#etags_array[@]} ]; then + log 2 "expected tag size of '$#', actual ${#etags_array[@]}" + return 1 + fi + while [ $# -gt 0 ]; do + if [ "$1" != "${etags_array[$idx]}" ]; then + log 2 "etag mismatch (expected '$1', actual ${etags_array[$idx]})" + return 1 + fi + ((idx++)) + shift + done + return 0 +} + check_part_list_rest() { if [ $# -lt 4 ]; then log 2 "'check_part_list_rest' requires bucket, file name, upload ID, expected count, etags" @@ -33,21 +55,10 @@ check_part_list_rest() { if [ "$4" == 0 ]; then return 0 fi - if ! etags=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/parts.txt" | tr '\n' ' ' 2>&1); then - log 2 "error retrieving etags: $etags" + if ! check_part_list_rest_etags "${@:5}"; then + log 2 "error checking etags" return 1 fi - read -ra etags_array <<< "$etags" - shift 4 - idx=0 - while [ $# -gt 0 ]; do - if [ "$1" != "${etags_array[$idx]}" ]; then - log 2 "etag mismatch (expected '$1', actual ${etags_array[$idx]})" - return 1 - fi - ((idx++)) - shift - done return 0 }