diff --git a/tests/rest_scripts/list_objects.sh b/tests/rest_scripts/list_objects.sh index f543163..55c1415 100755 --- a/tests/rest_scripts/list_objects.sh +++ b/tests/rest_scripts/list_objects.sh @@ -36,19 +36,19 @@ canonical_request="GET /$bucket_name " +queries="" if [ "$MARKER" != "" ]; then - add_parameter "canonical_request" "marker=$marker" + queries=$(add_parameter "$queries" "marker=$marker") fi if [ "$CONTINUATION_TOKEN" != "" ]; then - add_parameter "canonical_request" "continuation-token=$continuation_token" + queries=$(add_parameter "$queries" "continuation-token=$continuation_token") fi if [ "$version_two" != "FALSE" ]; then - add_parameter "canonical_request" "list-type=2" + queries=$(add_parameter "$queries" "list-type=2") fi if [ "$max_keys" -ne 0 ]; then - add_parameter "canonical_request" "max-keys=$max_keys" + queries=$(add_parameter "$queries" "max-keys=$max_keys") fi -first_param_added="false" canonical_request+=" host:$host @@ -61,19 +61,9 @@ create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"") url="'$AWS_ENDPOINT_URL/$bucket_name" -if [ "$MARKER" != "" ]; then - add_parameter "url" "marker=$marker" +if [ "$queries" != "" ]; then + url+="?$queries" fi -if [ "$CONTINUATION_TOKEN" != "" ]; then - add_parameter "url" "continuation-token=$continuation_token" -fi -if [ "$version_two" != "FALSE" ]; then - add_parameter "url" "list-type=2" -fi -if [ "$max_keys" -ne 0 ]; then - add_parameter "url" "max-keys=$max_keys" -fi -first_param_added="false" url+="'" curl_command+=("$url") curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" diff --git a/tests/rest_scripts/rest.sh b/tests/rest_scripts/rest.sh index 332de21..3ea3c44 100644 --- a/tests/rest_scripts/rest.sh +++ b/tests/rest_scripts/rest.sh @@ -70,13 +70,10 @@ add_parameter() { if [ "$#" -ne 2 ]; then return fi - if [ "$first_param_added" != "true" ]; then - if [ "$1" == "url" ]; then - eval "$1"+='?' - fi - eval "$1"+="$2" - first_param_added="true" - else - eval "$1"+="'&$2'" + current_string="$1" + if [ "$current_string" != "" ]; then + current_string+="&" fi + current_string+="$2" + echo "$current_string" } diff --git a/tests/rest_scripts/upload_part.sh b/tests/rest_scripts/upload_part.sh index 97e91c2..eff4c74 100755 --- a/tests/rest_scripts/upload_part.sh +++ b/tests/rest_scripts/upload_part.sh @@ -46,8 +46,6 @@ $payload_hash" create_canonical_hash_sts_and_signature -sleep 5 - curl_command+=(curl -isk -w "\"%{http_code}\"" "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\"" -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" -H "\"x-amz-content-sha256: $payload_hash\"" diff --git a/tests/rest_scripts/upload_part_copy.sh b/tests/rest_scripts/upload_part_copy.sh new file mode 100755 index 0000000..e849731 --- /dev/null +++ b/tests/rest_scripts/upload_part_copy.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" +# shellcheck disable=SC2153 +part_number="$PART_NUMBER" +# shellcheck disable=SC2153 +upload_id="$UPLOAD_ID" +# shellcheck disable=SC2153 +part_location=$PART_LOCATION + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") +aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} +# shellcheck disable=SC2034 +header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') +# shellcheck disable=SC2154 +canonical_request="PUT +/$bucket_name/$key +partNumber=$part_number&uploadId=$upload_id +host:$aws_endpoint_url_address +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-copy-source:$part_location +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-copy-source;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\"" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-copy-source: $part_location\"" +-H "\"x-amz-date: $current_date_time\"" +-o "\"$OUTPUT_FILE\"") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/setup.sh b/tests/setup.sh index 6ec5f82..8b61f22 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -14,9 +14,6 @@ # specific language governing permissions and limitations # under the License. -load ./bats-support/load -load ./bats-assert/load - source ./tests/env.sh source ./tests/report.sh source ./tests/setup_mc.sh @@ -68,11 +65,14 @@ delete_temp_log_if_exists() { # bats teardown function teardown() { # shellcheck disable=SC2154 - if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then - log 3 "error deleting bucket $BUCKET_ONE_NAME or contents" - fi - if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"; then - log 3 "error deleting bucket $BUCKET_TWO_NAME or contents" + if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ]; then + log 5 "deleting or clearing buckets" + if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then + log 3 "error deleting bucket $BUCKET_ONE_NAME or contents" + fi + if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"; then + log 3 "error deleting bucket $BUCKET_TWO_NAME or contents" + fi fi if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then log 3 "error deleting user $USERNAME_ONE" diff --git a/tests/teardown_static.sh b/tests/teardown_static.sh index bef762e..db0e46e 100755 --- a/tests/teardown_static.sh +++ b/tests/teardown_static.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2024 Versity Software # This file is licensed under the Apache License, Version 2.0 @@ -21,12 +21,13 @@ if ! base_setup; then log 2 "error starting versity to set up static buckets" exit 1 fi -if ! delete_bucket_recursive "s3" "$BUCKET_ONE_NAME"; then - log 2 "error creating static bucket one" -elif ! delete_bucket_recursive "s3" "$BUCKET_TWO_NAME"; then - log 2 "error creating static bucket two" +if ! delete_bucket_recursive "s3api" "$BUCKET_ONE_NAME"; then + log 2 "error deleting static bucket one" +elif ! delete_bucket_recursive "s3api" "$BUCKET_TWO_NAME"; then + log 2 "error deleting static bucket two" +else + log 4 "buckets deleted successfully" fi -log 4 "buckets deleted successfully" if ! stop_versity; then log 2 "error stopping versity" fi \ No newline at end of file diff --git a/tests/test_mc.sh b/tests/test_mc.sh index 54a7fb6..71a323d 100755 --- a/tests/test_mc.sh +++ b/tests/test_mc.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/test_common.sh source ./tests/setup.sh source ./tests/util/util_create_bucket.sh diff --git a/tests/test_mc_file_count.sh b/tests/test_mc_file_count.sh index c8be937..43808ed 100755 --- a/tests/test_mc_file_count.sh +++ b/tests/test_mc_file_count.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/test_common.sh export RUN_MC=true diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 1d07e7e..d8aaebf 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/commands/create_multipart_upload.sh source ./tests/commands/delete_object_tagging.sh source ./tests/commands/get_bucket_versioning.sh @@ -383,9 +386,6 @@ export RUN_USERS=true } @test "REST - bucket tagging - tags" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/932" - fi test_key="testKey" test_value="testValue" @@ -491,4 +491,34 @@ export RUN_USERS=true run list_objects_v1_check_nextmarker_empty "$BUCKET_ONE_NAME" assert_success -} \ No newline at end of file +} + +@test "REST - complete upload - invalid part" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1008" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test_file" + run create_large_file "$test_file" + assert_success + + run create_upload_finish_wrong_etag "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "REST - upload part copy" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test_file" + run create_large_file "$test_file" + assert_success + + run create_upload_part_copy_rest "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file" + assert_success + + run download_and_compare_file "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + assert_success +} diff --git a/tests/test_rest_acl.sh b/tests/test_rest_acl.sh index 2319d74..954bebc 100755 --- a/tests/test_rest_acl.sh +++ b/tests/test_rest_acl.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/commands/put_object.sh source ./tests/logger.sh source ./tests/setup.sh @@ -23,9 +26,6 @@ source ./tests/util/util_acl.sh export RUN_USERS=true @test "REST - get ACL" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/971" - fi run setup_bucket "s3api" "$BUCKET_ONE_NAME" assert_success diff --git a/tests/test_s3.sh b/tests/test_s3.sh index ae34a06..6af6bf1 100755 --- a/tests/test_s3.sh +++ b/tests/test_s3.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/test_common.sh source ./tests/util/util_file.sh diff --git a/tests/test_s3_file_count.sh b/tests/test_s3_file_count.sh index ef88d26..025b13d 100755 --- a/tests/test_s3_file_count.sh +++ b/tests/test_s3_file_count.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/test_common.sh source ./tests/util/util_file.sh diff --git a/tests/test_s3api_bucket.sh b/tests/test_s3api_bucket.sh index 0dfb556..fb2da08 100755 --- a/tests/test_s3api_bucket.sh +++ b/tests/test_s3api_bucket.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/setup.sh source ./tests/util/util.sh source ./tests/util/util_create_bucket.sh diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index d6025ad..fb382e4 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/setup.sh source ./tests/test_s3api_root_inner.sh source ./tests/util/util_file.sh diff --git a/tests/test_s3api_object.sh b/tests/test_s3api_object.sh index 4a18095..11c4322 100755 --- a/tests/test_s3api_object.sh +++ b/tests/test_s3api_object.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/setup.sh source ./tests/util/util.sh source ./tests/util/util_create_bucket.sh diff --git a/tests/test_s3api_policy.sh b/tests/test_s3api_policy.sh index c5380d5..0bfbf60 100755 --- a/tests/test_s3api_policy.sh +++ b/tests/test_s3api_policy.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/logger.sh source ./tests/setup.sh source ./tests/test_s3api_policy_bucket.sh diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 6ac5596..88d1d55 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/setup.sh source ./tests/test_common.sh source ./tests/test_common_acl.sh diff --git a/tests/test_s3cmd_file_count.sh b/tests/test_s3cmd_file_count.sh index 3dc492d..9efa0dc 100755 --- a/tests/test_s3cmd_file_count.sh +++ b/tests/test_s3cmd_file_count.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/test_common.sh export RUN_S3CMD=true diff --git a/tests/test_user_aws.sh b/tests/test_user_aws.sh index a08fd00..2583c56 100755 --- a/tests/test_user_aws.sh +++ b/tests/test_user_aws.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/test_user_common.sh source ./tests/util/util_users.sh source ./tests/commands/get_object.sh diff --git a/tests/test_user_s3cmd.sh b/tests/test_user_s3cmd.sh index 237d0eb..62d3a89 100755 --- a/tests/test_user_s3cmd.sh +++ b/tests/test_user_s3cmd.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +load ./bats-support/load +load ./bats-assert/load + source ./tests/test_user_common.sh export RUN_S3CMD=true diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index efbc8ed..5c992d7 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -211,16 +211,12 @@ bucket_cleanup() { # return 0 for success, 1 for error bucket_cleanup_if_bucket_exists() { log 6 "bucket_cleanup_if_bucket_exists" - if [ $# -ne 2 ]; then - log 2 "'bucket_cleanup_if_bucket_exists' requires client, bucket name" + if [ $# -lt 2 ]; then + log 2 "'bucket_cleanup_if_bucket_exists' requires client, bucket name, bucket known to exist (optional)" return 1 fi - if bucket_exists "$1" "$2"; then - if [ "$DELETE_BUCKETS_AFTER_TEST" == "false" ]; then - log 2 "skipping bucket cleanup/deletion" - return 0 - fi + if [ "$3" == "true" ] || bucket_exists "$1" "$2"; then if ! bucket_cleanup "$1" "$2"; then log 2 "error deleting bucket and/or contents" return 1 @@ -256,12 +252,16 @@ setup_bucket() { return 1 fi - if ! bucket_exists "$1" "$2" && [[ $RECREATE_BUCKETS == "false" ]]; then - log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user" - return 1 + bucket_exists="true" + if ! bucket_exists "$1" "$2"; then + if [[ $RECREATE_BUCKETS == "false" ]]; then + log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user" + return 1 + fi + bucket_exists="false" fi - if ! bucket_cleanup_if_bucket_exists "$1" "$2"; then + if ! bucket_cleanup_if_bucket_exists "$1" "$2" "$bucket_exists"; then log 2 "error deleting bucket or contents if they exist" return 1 fi diff --git a/tests/util/util_file.sh b/tests/util/util_file.sh index ad40326..f53863a 100644 --- a/tests/util/util_file.sh +++ b/tests/util/util_file.sh @@ -118,9 +118,7 @@ split_file() { fi local error - local split_result - error=$(split -a 1 -d -b "$part_size" "$1" "$1"-) || split_result=$? - if [[ $split_result -ne 0 ]]; then + if ! error=$(split -a 1 -d -b "$part_size" "$1" "$1"- 2>&1); then log 2 "error splitting file: $error" return 1 fi diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh index 4cd0eb6..c30a25e 100644 --- a/tests/util/util_multipart.sh +++ b/tests/util/util_multipart.sh @@ -34,7 +34,7 @@ multipart_upload_from_bucket() { } if ! create_multipart_upload "$1" "$2-copy"; then - log 2 "error running first multpart upload" + log 2 "error running first multipart upload" return 1 fi @@ -60,6 +60,25 @@ multipart_upload_from_bucket() { return 0 } +split_and_put_file() { + if [ $# -ne 4 ]; then + log 2 "'split_and_put_file' requires bucket, key, copy source, part count" + return 1 + fi + if ! split_file "$3" "$4"; then + log 2 "error splitting file" + return 1 + fi + for ((i=0;i<$4;i++)) { + log 5 "key: $2, file info: $(ls -l "$3"-"$i")" + if ! put_object "s3api" "$3-$i" "$1" "$2-$i"; then + log 2 "error copying object" + return 1 + fi + } + return 0 +} + multipart_upload_from_bucket_range() { if [ $# -ne 5 ]; then log 2 "multipart upload from bucket with range command requires bucket, copy source, key, part count, and range" @@ -189,3 +208,90 @@ run_and_verify_multipart_upload_with_valid_range() { fi return 0 } + +create_upload_part_copy_rest() { + if [ $# -ne 3 ]; then + log 2 "'run_and_verify_multipart_upload_with_valid_range' requires bucket, key, >20MB file" + return 1 + fi + if ! split_and_put_file "$1" "$2" "$3" 4; then + log 2 "error splitting and putting file" + return 1 + fi + if ! create_upload_and_get_id_rest "$1" "$2"; then + log 2 "error creating upload and getting ID" + return 1 + fi + parts_payload="" + for ((i=0; i<=3; i++)); do + part_number=$((i+1)) + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="$part_number" UPLOAD_ID="$upload_id" PART_LOCATION="$1/$2-$i" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part_copy.sh); then + # shellcheck disable=SC2154 + log 2 "error uploading part $i: $result" + return 1 + fi + log 5 "result: $result" + if [ "$result" != "200" ]; then + log 2 "error uploading part $i: $(cat "$TEST_FILE_FOLDER/response.txt")" + return 1 + fi + if ! etag=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/response.txt" 2>&1); then + log 2 "error retrieving etag: $etag" + return 1 + fi + parts_payload+="$etag$part_number" + done + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then + log 2 "error completing multipart upload: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")" + return 1 + fi + return 0 +} + +create_upload_finish_wrong_etag() { + if [ $# -ne 2 ]; then + log 2 "'create_upload_finish_wrong_etag' requires bucket, key" + return 1 + fi + + etag="gibberish" + part_number=1 + if ! create_upload_and_get_id_rest "$1" "$2"; then + log 2 "error creating upload and getting ID" + return 1 + fi + parts_payload="$etag$part_number" + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then + log 2 "error completing multipart upload: $result" + return 1 + fi + if [ "$result" != "400" ]; then + log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")" + return 1 + fi + if ! error=$(xmllint --xpath '//*[local-name()="Error"]' "$TEST_FILE_FOLDER/result.txt" 2>&1); then + log 2 "error retrieving error info: $error" + return 1 + fi + if ! check_xml_element <(echo "$error") "InvalidPart" "Code"; then + log 2 "code mismatch" + return 1 + fi + if ! check_xml_element <(echo "$error") "$upload_id" "UploadId"; then + log 2 "upload ID mismatch" + return 1 + fi + if ! check_xml_element <(echo "$error") "$part_number" "PartNumber"; then + log 2 "part number mismatch" + return 1 + fi + if ! check_xml_element <(echo "$error") "$etag" "ETag"; then + log 2 "etag mismatch" + return 1 + fi + return 0 +}