diff --git a/tests/commands/upload_part.sh b/tests/commands/upload_part.sh index da376ca..ec76245 100644 --- a/tests/commands/upload_part.sh +++ b/tests/commands/upload_part.sh @@ -31,3 +31,22 @@ upload_part() { fi export etag } + +upload_part_and_get_etag_rest() { + if [ $# -ne 5 ]; then + log 2 "'upload_part_rest' requires bucket name, key, part number, upload ID, part" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="$4" UPLOAD_ID="$3" DATA_FILE="$5" OUTPUT_FILE="$TEST_FILE_FOLDER/etag.txt" ./tests/rest_scripts/upload_part.sh); then + log 2 "error sending upload-part REST command: $result" + return 1 + fi + if [[ "$result" != "200" ]]; then + log 2 "upload-part command returned error $result: $(cat "$TEST_FILE_FOLDER/etag.txt")" + return 1 + fi + log 5 "$(cat "$TEST_FILE_FOLDER/etag.txt")" + etag=$(grep -i "etag" "$TEST_FILE_FOLDER/etag.txt" | awk '{print $2}' | tr -d '\r') + log 5 "etag: $etag" + return 0 +} diff --git a/tests/rest_scripts/abort_multipart_upload.sh b/tests/rest_scripts/abort_multipart_upload.sh index 4d5c610..a3a26ac 100755 --- a/tests/rest_scripts/abort_multipart_upload.sh +++ b/tests/rest_scripts/abort_multipart_upload.sh @@ -39,7 +39,7 @@ UNSIGNED-PAYLOAD" create_canonical_hash_sts_and_signature -curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "https://$host/$bucket_name/$key?uploadId=$upload_id" +curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id" -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" -H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" -H "\"x-amz-date: $current_date_time\"" diff --git a/tests/rest_scripts/complete_multipart_upload.sh b/tests/rest_scripts/complete_multipart_upload.sh new file mode 100755 index 0000000..b39a38e --- /dev/null +++ b/tests/rest_scripts/complete_multipart_upload.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" +# shellcheck disable=SC2153,SC2034 +upload_id="$UPLOAD_ID" +# shellcheck disable=SC2153 +parts="$PARTS" + +payload="$parts" +payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="POST +/$bucket_name/$key +uploadId=$UPLOAD_ID +host:$host +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: $payload_hash\"" +-H "\"x-amz-date: $current_date_time\"" +-H "\"Content-Type: application/xml\"" +-d "\"${payload//\"/\\\"}\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/list_multipart_uploads.sh b/tests/rest_scripts/list_multipart_uploads.sh new file mode 100755 index 0000000..038b77b --- /dev/null +++ b/tests/rest_scripts/list_multipart_uploads.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153,SC2034 +upload_id="$UPLOAD_ID" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +# shellcheck disable=SC2034 +canonical_request="GET +/$bucket_name/ +uploads= +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name/$key?uploads=" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 \ No newline at end of file diff --git a/tests/rest_scripts/upload_part.sh b/tests/rest_scripts/upload_part.sh new file mode 100755 index 0000000..97e91c2 --- /dev/null +++ b/tests/rest_scripts/upload_part.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" +# shellcheck disable=SC2153 +part_number="$PART_NUMBER" +# shellcheck disable=SC2153 +upload_id="$UPLOAD_ID" +# shellcheck disable=SC2153 +data=$DATA_FILE + +payload_hash="$(sha256sum "$data" | awk '{print $1}')" + + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} + # shellcheck disable=SC2034 + header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') + # shellcheck disable=SC2154 + canonical_request="PUT +/$bucket_name/$key +partNumber=$part_number&uploadId=$upload_id +host:$aws_endpoint_url_address +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + +create_canonical_hash_sts_and_signature + +sleep 5 + +curl_command+=(curl -isk -w "\"%{http_code}\"" "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\"" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: $payload_hash\"" +-H "\"x-amz-date: $current_date_time\"" +-o "\"$OUTPUT_FILE\"" +-T "\"$data\"") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/setup.sh b/tests/setup.sh index 8eb1ece..25cd8fb 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -62,10 +62,10 @@ teardown() { echo "**********************************************************************************" fi # shellcheck disable=SC2154 - if ! delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME"; then + if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then log 3 "error deleting bucket $BUCKET_ONE_NAME or contents" fi - if ! delete_bucket_or_contents_if_exists "s3api" "$BUCKET_TWO_NAME"; then + if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"; then log 3 "error deleting bucket $BUCKET_TWO_NAME or contents" fi if [ "$REMOVE_TEST_FILE_FOLDER" == "true" ]; then diff --git a/tests/test_aws_root_inner.sh b/tests/test_aws_root_inner.sh index 5c0c2fa..87af7f5 100755 --- a/tests/test_aws_root_inner.sh +++ b/tests/test_aws_root_inner.sh @@ -87,7 +87,7 @@ test_create_multipart_upload_properties_aws_root() { run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1 assert_success - run delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME" + run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME" assert_success # in static bucket config, bucket will still exist if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then @@ -360,7 +360,7 @@ test_retention_bypass_aws_root() { legal_hold_retention_setup() { assert [ $# -eq 3 ] - run delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME" + run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME" assert_success run setup_user "$1" "$2" "user" diff --git a/tests/test_common.sh b/tests/test_common.sh index bfc0950..f252933 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -65,7 +65,7 @@ test_common_multipart_upload() { run download_and_compare_file "$1" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" assert_success - delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" + bucket_cleanup "$1" "$BUCKET_ONE_NAME" delete_test_files $bucket_file } @@ -85,7 +85,7 @@ test_common_create_delete_bucket() { run bucket_exists "$1" "$BUCKET_ONE_NAME" assert_success - run delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" + run bucket_cleanup "$1" "$BUCKET_ONE_NAME" assert_success } diff --git a/tests/test_rest.sh b/tests/test_rest.sh index da088a8..dd303cd 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -32,6 +32,7 @@ source ./tests/util.sh source ./tests/util_legal_hold.sh source ./tests/util_list_buckets.sh source ./tests/util_list_objects.sh +source ./tests/util_list_parts.sh source ./tests/util_lock_config.sh source ./tests/util_rest.sh source ./tests/util_tags.sh @@ -117,7 +118,7 @@ source ./tests/util_versioning.sh test_key="TestKey" test_value="TestValue" - run delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME" + run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME" assert_success # in static bucket config, bucket will still exist if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then @@ -169,7 +170,7 @@ source ./tests/util_versioning.sh run check_no_object_lock_config_rest "$BUCKET_ONE_NAME" assert_success - run delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME" + run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME" assert_success # in static bucket config, bucket will still exist @@ -291,3 +292,26 @@ source ./tests/util_versioning.sh run create_abort_multipart_upload_rest "$BUCKET_ONE_NAME" "$test_file" assert_success } + +@test "REST - multipart upload create, list parts" { + test_file="test_file" + + run create_large_file "$test_file" + assert_success + + run split_file "$TEST_FILE_FOLDER/$test_file" 4 + assert_success + + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run upload_check_parts "$BUCKET_ONE_NAME" "$test_file" \ + "$TEST_FILE_FOLDER/$test_file-0" "$TEST_FILE_FOLDER/$test_file-1" "$TEST_FILE_FOLDER/$test_file-2" "$TEST_FILE_FOLDER/$test_file-3" + assert_success + + run get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + assert_success + + run compare_files "$TEST_FILE_FOLDER/$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + assert_success +} diff --git a/tests/test_s3api.sh b/tests/test_s3api.sh index 2fa0209..f9f836c 100755 --- a/tests/test_s3api.sh +++ b/tests/test_s3api.sh @@ -283,7 +283,7 @@ export RUN_USERS=true fi [[ object_size -eq $((range_max*4+4)) ]] || fail "object size mismatch ($object_size, $((range_max*4+4)))" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files $bucket_file } @@ -314,7 +314,7 @@ export RUN_USERS=true key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1) || fail "error getting key from object list: $key" [[ $key == "$folder_name/$object_name" ]] || fail "key doesn't match (expected $key, actual $folder_name/$object_name)" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files $folder_name } @@ -350,7 +350,7 @@ export RUN_USERS=true log 5 "INFO: $bucket_info" region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region" [[ $region != "" ]] || fail "empty bucket region" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" } @test "test_retention_bypass" { @@ -364,7 +364,7 @@ export RUN_USERS=true head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$? [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned" [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" } @test "test_add_object_metadata" { @@ -388,7 +388,7 @@ export RUN_USERS=true [[ $key == "$test_key" ]] || fail "keys doesn't match (expected $key, actual \"$test_key\")" [[ $value == "$test_value" ]] || fail "values doesn't match (expected $value, actual \"$test_value\")" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$object_one" } @@ -410,7 +410,7 @@ export RUN_USERS=true run get_and_check_object_lock_config "$bucket_name" "$enabled" "$governance" "$days" assert_success "error getting and checking object lock config" - delete_bucket_or_contents "aws" "$bucket_name" + bucket_cleanup "aws" "$bucket_name" } @test "test_ls_directory_object" { diff --git a/tests/test_s3api_policy.sh b/tests/test_s3api_policy.sh index 7f1d52b..91e793d 100755 --- a/tests/test_s3api_policy.sh +++ b/tests/test_s3api_policy.sh @@ -187,14 +187,15 @@ test_s3api_policy_invalid_action() { run setup_bucket "s3api" "$BUCKET_ONE_NAME" assert_success - check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" || fail "policy not empty" + run check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" + assert_success if put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"; then fail "put succeeded despite malformed policy" fi # shellcheck disable=SC2154 [[ "$put_bucket_policy_error" == *"MalformedPolicy"*"invalid action"* ]] || fail "invalid policy error: $put_bucket_policy_error" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$policy_file" } @@ -236,7 +237,7 @@ test_s3api_policy_get_object_with_user() { run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password" assert_success - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" } test_s3api_policy_get_object_specific_file() { @@ -273,7 +274,7 @@ test_s3api_policy_get_object_specific_file() { fi # shellcheck disable=SC2154 [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" } test_s3api_policy_get_object_file_wildcard() { @@ -314,7 +315,7 @@ test_s3api_policy_get_object_file_wildcard() { fi [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" } test_s3api_policy_get_object_folder_wildcard() { @@ -335,17 +336,23 @@ test_s3api_policy_get_object_folder_wildcard() { action="s3:GetObject" resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*" - setup_user "$username" "$password" "user" || fail "error creating user" + run setup_user "$username" "$password" "user" + assert_success - setup_bucket "s3api" "$BUCKET_ONE_NAME" - setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" - put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy" + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success - put_object "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" || fail "error copying object to bucket" + run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" + assert_success - download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password" || fail "error downloading and comparing file" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files "$test_folder/$test_file" "$policy_file" + run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" + assert_success + + run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password" + assert_success } test_s3api_policy_allow_deny() { @@ -374,7 +381,7 @@ test_s3api_policy_allow_deny() { fi [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$test_file" "$test_file-copy" "$policy_file" } @@ -406,7 +413,7 @@ test_s3api_policy_deny() { fail "able to get object despite deny statement" fi [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$test_file_one" "$test_file_two" "$test_file_one-copy" "$test_file_two-copy" "$policy_file" } @@ -445,7 +452,7 @@ test_s3api_policy_put_wildcard() { fi [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error" download_and_compare_file "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$TEST_FILE_FOLDER/$test_file-copy" || fail "files don't match" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$test_folder/$test_file" "$test_file-copy" "$policy_file" } @@ -481,7 +488,7 @@ test_s3api_policy_delete() { # shellcheck disable=SC2154 [[ "$delete_object_error" == *"Access Denied"* ]] || fail "invalid delete object error: $delete_object_error" delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$username" "$password" || fail "error deleting object despite permissions" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$test_file_one" "$test_file_two" "$policy_file" } @@ -515,7 +522,7 @@ test_s3api_policy_get_bucket_policy() { log 5 "ORIG: $(cat "$TEST_FILE_FOLDER/$policy_file")" log 5 "COPY: $(cat "$TEST_FILE_FOLDER/$policy_file-copy")" compare_files "$TEST_FILE_FOLDER/$policy_file" "$TEST_FILE_FOLDER/$policy_file-copy" || fail "policies not equal" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$policy_file" "$policy_file-copy" } @@ -560,7 +567,7 @@ test_s3api_policy_list_multipart_uploads() { log 5 "$uploads" upload_key=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -r ".Uploads[0].Key" 2>&1) || fail "error parsing upload key from uploads message: $upload_key" [[ $upload_key == "$test_file" ]] || fail "upload key doesn't match file marked as being uploaded" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$policy_file" "$test_file" } @@ -597,7 +604,7 @@ test_s3api_policy_put_bucket_policy() { log 5 "ORIG: $(cat "$TEST_FILE_FOLDER/$policy_file_two")" log 5 "COPY: $(cat "$TEST_FILE_FOLDER/$policy_file-copy")" compare_files "$TEST_FILE_FOLDER/$policy_file_two" "$TEST_FILE_FOLDER/$policy_file-copy" || fail "policies not equal" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$policy_file" "$policy_file_two" "$policy_file-copy" } @@ -625,7 +632,7 @@ test_s3api_policy_delete_bucket_policy() { setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy" delete_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "unable to delete bucket policy" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$policy_file" } @@ -700,7 +707,7 @@ test_s3api_policy_abort_multipart_upload() { put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy" abort_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$upload_id" "$username" "$password" || fail "error aborting multipart upload despite permissions" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$policy_file" "$test_file" } @@ -736,7 +743,7 @@ test_s3api_policy_two_principals() { assert_success "error getting object with user $USERNAME_TWO" delete_test_files "$test_file" "$policy_file" "$TEST_FILE_FOLDER/copy_one" "$TEST_FILE_FOLDER/copy_two" - delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME" + bucket_cleanup "s3api" "$BUCKET_ONE_NAME" } test_s3api_policy_put_bucket_tagging() { @@ -762,7 +769,7 @@ test_s3api_policy_put_bucket_tagging() { get_and_check_bucket_tags "$BUCKET_ONE_NAME" "$tag_key" "$tag_value" - delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME" + bucket_cleanup "s3api" "$BUCKET_ONE_NAME" } test_s3api_policy_put_acl() { @@ -805,7 +812,7 @@ test_s3api_policy_put_acl() { id=$(echo "$second_grantee" | jq -r ".ID" 2>&1) || fail "error getting ID: $id" [[ $id == "all-users" ]] || fail "unexpected ID: $id" fi - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + bucket_cleanup "aws" "$BUCKET_ONE_NAME" delete_test_files "$policy_file" } @@ -838,7 +845,7 @@ test_s3api_policy_get_bucket_tagging() { run get_and_check_bucket_tags_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$tag_key" "$tag_value" assert_success "get and check bucket tags failed" - delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME" + bucket_cleanup "s3api" "$BUCKET_ONE_NAME" delete_test_files "$policy_file" } @@ -869,6 +876,6 @@ test_s3api_policy_list_upload_parts() { run create_upload_and_test_parts_listing "$test_file" "$policy_file" assert_success "error creating upload and testing parts listing" - delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME" + bucket_cleanup "s3api" "$BUCKET_ONE_NAME" delete_test_files "$policy_file" "$test_file" } diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 2947eb3..0072f27 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -52,7 +52,7 @@ export RUN_USERS=true [[ "$bucket_create_error" == *"just the bucket name"* ]] || fail "unexpected error: $bucket_create_error" - delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME" + bucket_cleanup "s3cmd" "$BUCKET_ONE_NAME" } # delete-bucket - test_create_delete_bucket @@ -119,7 +119,7 @@ export RUN_USERS=true head_bucket "s3cmd" "$BUCKET_ONE_NAME" [[ $bucket_info == *"s3://$BUCKET_ONE_NAME"* ]] || fail "failure to retrieve correct bucket info: $bucket_info" - delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME" + bucket_cleanup "s3cmd" "$BUCKET_ONE_NAME" } @test "test_get_bucket_info_doesnt_exist_s3cmd" { @@ -129,7 +129,7 @@ export RUN_USERS=true head_bucket "s3cmd" "$BUCKET_ONE_NAME"a || local info_result=$? [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned" [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info" - delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME" + bucket_cleanup "s3cmd" "$BUCKET_ONE_NAME" } @test "test_ls_directory_object" { diff --git a/tests/test_user_aws.sh b/tests/test_user_aws.sh index bfa28c3..2ad3e26 100755 --- a/tests/test_user_aws.sh +++ b/tests/test_user_aws.sh @@ -136,7 +136,7 @@ export RUN_USERS=true fi # shellcheck disable=SC2154 [[ "$get_object_error" == *"NoSuchKey"* ]] || fail "unexpected error message: $get_object_error" - delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME" + bucket_cleanup "s3api" "$BUCKET_ONE_NAME" delete_test_files "$test_file" "$test_file-copy" } diff --git a/tests/test_user_common.sh b/tests/test_user_common.sh index 75fa401..6a0fdb9 100755 --- a/tests/test_user_common.sh +++ b/tests/test_user_common.sh @@ -87,7 +87,7 @@ test_user_user() { password="$PASSWORD_ONE" setup_user "$username" "$password" "user" || fail "error setting up user" - delete_bucket_or_contents_if_exists "aws" "versity-gwtest-user-bucket" + bucket_cleanup_if_bucket_exists "aws" "versity-gwtest-user-bucket" run setup_bucket "aws" "$BUCKET_ONE_NAME" assert_success @@ -131,7 +131,7 @@ test_userplus_operation() { username="$USERNAME_ONE" password="$PASSWORD_ONE" - delete_bucket_or_contents_if_exists "aws" "versity-gwtest-userplus-bucket" + bucket_cleanup_if_bucket_exists "aws" "versity-gwtest-userplus-bucket" setup_user "$username" "$password" "userplus" || fail "error creating user '$username'" run setup_bucket "aws" "$BUCKET_ONE_NAME" diff --git a/tests/util.sh b/tests/util.sh index a3e0c75..a72c24e 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/util_bucket.sh source ./tests/util_create_bucket.sh source ./tests/util_mc.sh source ./tests/util_multipart.sh @@ -46,46 +47,6 @@ source ./tests/commands/upload_part_copy.sh source ./tests/commands/upload_part.sh source ./tests/util_users.sh -# recursively delete an AWS bucket -# param: client, bucket name -# fail if error -delete_bucket_recursive() { - log 6 "delete_bucket_recursive" - if [ $# -ne 2 ]; then - log 2 "'delete_bucket_recursive' requires client, bucket name" - return 1 - fi - - local exit_code=0 - local error - if [[ $1 == 's3' ]]; then - error=$(aws --no-verify-ssl s3 rb s3://"$2" --force 2>&1) || exit_code="$?" - elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then - if ! delete_bucket_recursive_s3api "$2"; then - log 2 "error deleting bucket recursively (s3api)" - return 1 - fi - return 0 - elif [[ $1 == "s3cmd" ]]; then - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" --recursive 2>&1) || exit_code="$?" - elif [[ $1 == "mc" ]]; then - error=$(delete_bucket_recursive_mc "$2" 2>&1) || exit_code="$?" - else - log 2 "invalid client '$1'" - return 1 - fi - - if [ $exit_code -ne 0 ]; then - if [[ "$error" == *"The specified bucket does not exist"* ]]; then - return 0 - else - log 2 "error deleting bucket recursively: $error" - return 1 - fi - fi - return 0 -} - # params: bucket name # return 0 for success, 1 for error add_governance_bypass_policy() { @@ -200,56 +161,6 @@ check_object_lock_config() { return 0 } -# restore bucket to pre-test state (or prep for deletion) -# param: bucket name -# return 0 on success, 1 on error -clear_bucket_s3api() { - log 6 "clear_bucket_s3api" - if [ $# -ne 1 ]; then - log 2 "'clear_bucket_s3api' requires bucket name" - return 1 - fi - - if [[ $LOG_LEVEL_INT -ge 5 ]]; then - if ! log_bucket_policy "$1"; then - log 2 "error logging bucket policy" - return 1 - fi - fi - - if ! check_object_lock_config "$1"; then - log 2 "error checking object lock config" - return 1 - fi - - if [[ "$DIRECT" != "true" ]] && ! add_governance_bypass_policy "$1"; then - log 2 "error adding governance bypass policy" - return 1 - fi - - if ! list_and_delete_objects "$1"; then - log 2 "error listing and deleting objects" - return 1 - fi - - #run check_ownership_rule_and_reset_acl "$1" - #assert_success "error checking ownership rule and resetting acl" - - if [[ $lock_config_exists == true ]] && ! put_object_lock_configuration_disabled "$1"; then - log 2 "error disabling object lock config" - return 1 - fi - #if ! put_bucket_versioning "s3api" "$1" "Suspended"; then - # log 2 "error suspending bucket versioning" - # return 1 - #fi - - #if ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID"; then - # log 2 "error changing bucket owner back to root" - # return 1 - #fi -} - # params: bucket, object name # return 0 for success, 1 for error clear_object_in_bucket() { @@ -321,77 +232,6 @@ log_worm_protection() { log 5 "RETENTION: $retention" } -# params: bucket name -# return 0 if able to delete recursively, 1 if not -delete_bucket_recursive_s3api() { - log 6 "delete_bucket_recursive_s3api" - if [ $# -ne 1 ]; then - log 2 "'delete_bucket_recursive_s3api' requires bucket name" - return 1 - fi - - if ! clear_bucket_s3api "$1"; then - log 2 "error clearing bucket (s3api)" - return 1 - fi - - if ! delete_bucket 's3api' "$1"; then - log 2 "error deleting bucket" - return 1 - fi - return 0 -} - -# params: client, bucket name -# return 0 on success, 1 on error -delete_bucket_contents() { - log 6 "delete_bucket_contents" - if [ $# -ne 2 ]; then - log 2 "'delete_bucket_contents' requires client, bucket name" - return 1 - fi - - local exit_code=0 - local error - if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then - if ! clear_bucket_s3api "$2"; then - log 2 "error clearing bucket (s3api)" - return 1 - fi - elif [[ $1 == "s3cmd" ]]; then - delete_bucket_recursive "s3cmd" "$1" - elif [[ $1 == "mc" ]]; then - delete_bucket_recursive "mc" "$1" - elif [[ $1 == "s3" ]]; then - delete_bucket_recursive "s3" "$1" - else - log 2 "unrecognized client: '$1'" - return 1 - fi - return 0 -} - -# check if bucket exists -# param: bucket name -# return 0 for true, 1 for false, 2 for error -bucket_exists() { - if [ $# -ne 2 ]; then - log 2 "bucket_exists command requires client, bucket name" - return 2 - fi - local exists=0 - head_bucket "$1" "$2" || exists=$? - # shellcheck disable=SC2181 - if [ $exists -ne 0 ] && [ $exists -ne 1 ]; then - log 2 "unexpected error checking if bucket exists" - return 2 - fi - if [ $exists -eq 0 ]; then - return 0 - fi - return 1 -} - # param: bucket name # return 1 for failure, 0 for success get_object_ownership_rule_and_update_acl() { @@ -410,126 +250,6 @@ get_object_ownership_rule_and_update_acl() { fi } -# params: client, bucket name -# return 0 for success, 1 for error -delete_bucket_or_contents() { - log 6 "delete_bucket_or_contents" - if [ $# -ne 2 ]; then - log 2 "'delete_bucket_or_contents' requires client, bucket name" - return 1 - fi - if [[ $RECREATE_BUCKETS == "false" ]]; then - if ! delete_bucket_contents "$1" "$2"; then - log 2 "error deleting bucket contents" - return 1 - fi - - if ! delete_bucket_policy "$1" "$2"; then - log 2 "error deleting bucket policy" - return 1 - fi - - if ! get_object_ownership_rule_and_update_acl "$2"; then - log 2 "error getting object ownership rule and updating ACL" - return 1 - fi - - if ! abort_all_multipart_uploads "$2"; then - log 2 "error aborting all multipart uploads" - return 1 - fi - - if [ "$RUN_USERS" == "true" ] && ! reset_bucket_owner "$2"; then - log 2 "error resetting bucket owner" - return 1 - fi - - log 5 "bucket contents, policy, ACL deletion success" - return 0 - fi - if ! delete_bucket_recursive "$1" "$2"; then - log 2 "error with recursive bucket delete" - return 1 - fi - log 5 "bucket deletion success" - return 0 -} - -# params: client, bucket name -# return 0 for success, 1 for error -delete_bucket_or_contents_if_exists() { - log 6 "delete_bucket_or_contents_if_exists" - if [ $# -ne 2 ]; then - log 2 "'delete_bucket_or_contents_if_exists' requires client, bucket name" - return 1 - fi - - if bucket_exists "$1" "$2"; then - if ! delete_bucket_or_contents "$1" "$2"; then - log 2 "error deleting bucket and/or contents" - return 1 - fi - log 5 "bucket and/or bucket data deletion success" - return 0 - fi - return 0 -} - -# params: client, bucket name(s) -# return 0 for success, 1 for failure -setup_buckets() { - if [ $# -lt 2 ]; then - log 2 "'setup_buckets' command requires client, bucket names" - return 1 - fi - for name in "${@:2}"; do - if ! setup_bucket "$1" "$name"; then - log 2 "error setting up bucket $name" - return 1 - fi - done - return 0 -} - -# params: client, bucket name -# return 0 on successful setup, 1 on error -setup_bucket() { - log 6 "setup_bucket" - if [ $# -ne 2 ]; then - log 2 "'setup_bucket' requires client, bucket name" - return 1 - fi - - if ! bucket_exists "$1" "$2" && [[ $RECREATE_BUCKETS == "false" ]]; then - log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user" - return 1 - fi - - if ! delete_bucket_or_contents_if_exists "$1" "$2"; then - log 2 "error deleting bucket or contents if they exist" - return 1 - fi - - log 5 "util.setup_bucket: command type: $1, bucket name: $2" - if [[ $RECREATE_BUCKETS == "true" ]]; then - if ! create_bucket "$1" "$2"; then - log 2 "error creating bucket" - return 1 - fi - else - log 5 "skipping bucket re-creation" - fi - - if [[ $1 == "s3cmd" ]]; then - log 5 "putting bucket ownership controls" - if bucket_exists "s3cmd" "$2" && ! put_bucket_ownership_controls "$2" "BucketOwnerPreferred"; then - log 2 "error putting bucket ownership controls" - return 1 - fi - fi - return 0 -} - # check if object exists on S3 via gateway # param: command, object path # return 0 for true, 1 for false, 2 for error @@ -659,27 +379,6 @@ remove_insecure_request_warning() { export parsed_output } -# check if bucket info can be retrieved -# param: path of bucket or folder -# return 0 for yes, 1 for no, 2 for error -bucket_is_accessible() { - if [ $# -ne 1 ]; then - echo "bucket accessibility check missing bucket name" - return 2 - fi - local exit_code=0 - local error - error=$(aws --no-verify-ssl s3api head-bucket --bucket "$1" 2>&1) || exit_code="$?" - if [ $exit_code -eq 0 ]; then - return 0 - fi - if [[ "$error" == *"500"* ]]; then - return 1 - fi - echo "Error checking bucket accessibility: $error" - return 2 -} - # check if object info (etag) is accessible # param: path of object # return 0 for yes, 1 for no, 2 for error diff --git a/tests/util_aws.sh b/tests/util_aws.sh index e2ff67b..5f9b99f 100644 --- a/tests/util_aws.sh +++ b/tests/util_aws.sh @@ -34,22 +34,21 @@ abort_all_multipart_uploads() { log 5 "Modified upload list: ${modified_upload_list[*]}" has_uploads=$(echo "${modified_upload_list[*]}" | jq 'has("Uploads")') - if [[ $has_uploads != false ]]; then - lines=$(echo "${modified_upload_list[*]}" | jq -r '.Uploads[] | "--key \(.Key) --upload-id \(.UploadId)"') || lines_result=$? - if [[ $lines_result -ne 0 ]]; then - echo "error getting lines for multipart upload delete: $lines" + if [[ $has_uploads == false ]]; then + return 0 + fi + if ! lines=$(echo "${modified_upload_list[*]}" | jq -r '.Uploads[] | "--key \(.Key) --upload-id \(.UploadId)"' 2>&1); then + log 2 "error getting lines for multipart upload delete: $lines" + return 1 + fi + + log 5 "$lines" + while read -r line; do + # shellcheck disable=SC2086 + if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" $line 2>&1); then + echo "error aborting multipart upload: $error" return 1 fi - - log 5 "$lines" - while read -r line; do - # shellcheck disable=SC2086 - error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" $line 2>&1) || abort_result=$? - if [[ $abort_result -ne 0 ]]; then - echo "error aborting multipart upload: $error" - return 1 - fi - done <<< "$lines" - fi + done <<< "$lines" return 0 } \ No newline at end of file diff --git a/tests/util_bucket.sh b/tests/util_bucket.sh new file mode 100644 index 0000000..88f57cd --- /dev/null +++ b/tests/util_bucket.sh @@ -0,0 +1,304 @@ +#!/usr/bin/env bash + +# recursively delete an AWS bucket +# param: client, bucket name +# fail if error +delete_bucket_recursive() { + log 6 "delete_bucket_recursive" + if [ $# -ne 2 ]; then + log 2 "'delete_bucket_recursive' requires client, bucket name" + return 1 + fi + + local exit_code=0 + local error + if [[ $1 == 's3' ]]; then + error=$(aws --no-verify-ssl s3 rb s3://"$2" --force 2>&1) || exit_code="$?" + elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then + if ! delete_bucket_recursive_s3api "$2"; then + log 2 "error deleting bucket recursively (s3api)" + return 1 + fi + return 0 + elif [[ $1 == "s3cmd" ]]; then + error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" --recursive 2>&1) || exit_code="$?" + elif [[ $1 == "mc" ]]; then + error=$(delete_bucket_recursive_mc "$2" 2>&1) || exit_code="$?" + else + log 2 "invalid client '$1'" + return 1 + fi + + if [ $exit_code -ne 0 ]; then + if [[ "$error" == *"The specified bucket does not exist"* ]]; then + return 0 + else + log 2 "error deleting bucket recursively: $error" + return 1 + fi + fi + return 0 +} + +# restore bucket to pre-test state (or prep for deletion) +# param: bucket name +# return 0 on success, 1 on error +clear_bucket_s3api() { + log 6 "clear_bucket_s3api" + if [ $# -ne 1 ]; then + log 2 "'clear_bucket_s3api' requires bucket name" + return 1 + fi + + if [[ $LOG_LEVEL_INT -ge 5 ]]; then + if ! log_bucket_policy "$1"; then + log 2 "error logging bucket policy" + return 1 + fi + fi + + if ! check_object_lock_config "$1"; then + log 2 "error checking object lock config" + return 1 + fi + + if [[ "$DIRECT" != "true" ]] && ! add_governance_bypass_policy "$1"; then + log 2 "error adding governance bypass policy" + return 1 + fi + + if ! list_and_delete_objects "$1"; then + log 2 "error listing and deleting objects" + return 1 + fi + + #run check_ownership_rule_and_reset_acl "$1" + #assert_success "error checking ownership rule and resetting acl" + + # shellcheck disable=SC2154 + if [[ $lock_config_exists == true ]] && ! put_object_lock_configuration_disabled "$1"; then + log 2 "error disabling object lock config" + return 1 + fi + #if ! put_bucket_versioning "s3api" "$1" "Suspended"; then + # log 2 "error suspending bucket versioning" + # return 1 + #fi + + #if ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID"; then + # log 2 "error changing bucket owner back to root" + # return 1 + #fi +} + +# params: bucket name +# return 0 if able to delete recursively, 1 if not +delete_bucket_recursive_s3api() { + log 6 "delete_bucket_recursive_s3api" + if [ $# -ne 1 ]; then + log 2 "'delete_bucket_recursive_s3api' requires bucket name" + return 1 + fi + + if ! clear_bucket_s3api "$1"; then + log 2 "error clearing bucket (s3api)" + return 1 + fi + + if ! delete_bucket 's3api' "$1"; then + log 2 "error deleting bucket" + return 1 + fi + return 0 +} + +# params: client, bucket name +# return 0 on success, 1 on error +delete_bucket_contents() { + log 6 "delete_bucket_contents" + if [ $# -ne 2 ]; then + log 2 "'delete_bucket_contents' requires client, bucket name" + return 1 + fi + + local exit_code=0 + local error + if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then + if ! clear_bucket_s3api "$2"; then + log 2 "error clearing bucket (s3api)" + return 1 + fi + elif [[ $1 == "s3cmd" ]]; then + delete_bucket_recursive "s3cmd" "$1" + elif [[ $1 == "mc" ]]; then + delete_bucket_recursive "mc" "$1" + elif [[ $1 == "s3" ]]; then + delete_bucket_recursive "s3" "$1" + else + log 2 "unrecognized client: '$1'" + return 1 + fi + return 0 +} + +# check if bucket exists +# param: bucket name +# return 0 for true, 1 for false, 2 for error +bucket_exists() { + if [ $# -ne 2 ]; then + log 2 "bucket_exists command requires client, bucket name" + return 2 + fi + local exists=0 + head_bucket "$1" "$2" || exists=$? + # shellcheck disable=SC2181 + if [ $exists -ne 0 ] && [ $exists -ne 1 ]; then + log 2 "unexpected error checking if bucket exists" + return 2 + fi + if [ $exists -eq 0 ]; then + return 0 + fi + return 1 +} + +# params: client, bucket name +# return 0 for success, 1 for error +bucket_cleanup() { + log 6 "bucket_cleanup" + if [ $# -ne 2 ]; then + log 2 "'bucket_cleanup' requires client, bucket name" + return 1 + fi + if [[ $RECREATE_BUCKETS == "false" ]]; then + if ! delete_bucket_contents "$1" "$2"; then + log 2 "error deleting bucket contents" + return 1 + fi + + if ! delete_bucket_policy "$1" "$2"; then + log 2 "error deleting bucket policy" + return 1 + fi + + if ! get_object_ownership_rule_and_update_acl "$2"; then + log 2 "error getting object ownership rule and updating ACL" + return 1 + fi + + if ! abort_all_multipart_uploads "$2"; then + log 2 "error aborting all multipart uploads" + return 1 + fi + + if [ "$RUN_USERS" == "true" ] && ! reset_bucket_owner "$2"; then + log 2 "error resetting bucket owner" + return 1 + fi + + log 5 "bucket contents, policy, ACL deletion success" + return 0 + fi + if ! delete_bucket_recursive "$1" "$2"; then + log 2 "error with recursive bucket delete" + return 1 + fi + log 5 "bucket deletion success" + return 0 +} + +# params: client, bucket name +# return 0 for success, 1 for error +bucket_cleanup_if_bucket_exists() { + log 6 "bucket_cleanup_if_bucket_exists" + if [ $# -ne 2 ]; then + log 2 "'bucket_cleanup_if_bucket_exists' requires client, bucket name" + return 1 + fi + + if bucket_exists "$1" "$2"; then + if ! bucket_cleanup "$1" "$2"; then + log 2 "error deleting bucket and/or contents" + return 1 + fi + log 5 "bucket and/or bucket data deletion success" + return 0 + fi + return 0 +} + +# params: client, bucket name(s) +# return 0 for success, 1 for failure +setup_buckets() { + if [ $# -lt 2 ]; then + log 2 "'setup_buckets' command requires client, bucket names" + return 1 + fi + for name in "${@:2}"; do + if ! setup_bucket "$1" "$name"; then + log 2 "error setting up bucket $name" + return 1 + fi + done + return 0 +} + +# params: client, bucket name +# return 0 on successful setup, 1 on error +setup_bucket() { + log 6 "setup_bucket" + if [ $# -ne 2 ]; then + log 2 "'setup_bucket' requires client, bucket name" + return 1 + fi + + if ! bucket_exists "$1" "$2" && [[ $RECREATE_BUCKETS == "false" ]]; then + log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user" + return 1 + fi + + if ! bucket_cleanup_if_bucket_exists "$1" "$2"; then + log 2 "error deleting bucket or contents if they exist" + return 1 + fi + + log 5 "util.setup_bucket: command type: $1, bucket name: $2" + if [[ $RECREATE_BUCKETS == "true" ]]; then + if ! create_bucket "$1" "$2"; then + log 2 "error creating bucket" + return 1 + fi + else + log 5 "skipping bucket re-creation" + fi + + if [[ $1 == "s3cmd" ]]; then + log 5 "putting bucket ownership controls" + if bucket_exists "s3cmd" "$2" && ! put_bucket_ownership_controls "$2" "BucketOwnerPreferred"; then + log 2 "error putting bucket ownership controls" + return 1 + fi + fi + return 0 +} + +# check if bucket info can be retrieved +# param: path of bucket or folder +# return 0 for yes, 1 for no, 2 for error +bucket_is_accessible() { + if [ $# -ne 1 ]; then + echo "bucket accessibility check missing bucket name" + return 2 + fi + local exit_code=0 + local error + error=$(aws --no-verify-ssl s3api head-bucket --bucket "$1" 2>&1) || exit_code="$?" + if [ $exit_code -eq 0 ]; then + return 0 + fi + if [[ "$error" == *"500"* ]]; then + return 1 + fi + echo "Error checking bucket accessibility: $error" + return 2 +} diff --git a/tests/util_list_parts.sh b/tests/util_list_parts.sh new file mode 100644 index 0000000..e069ee4 --- /dev/null +++ b/tests/util_list_parts.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +check_part_list_rest() { + if [ $# -lt 4 ]; then + log 2 "'check_part_list_rest' requires bucket, file name, upload ID, expected count, etags" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/parts.txt" ./tests/rest_scripts/list_parts.sh); then + log 2 "error listing multipart upload parts: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "list-parts returned response code: $result, reply: $(cat "$TEST_FILE_FOLDER/parts.txt")" + return 1 + fi + log 5 "parts list: $(cat "$TEST_FILE_FOLDER/parts.txt")" + if ! parts_upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/parts.txt" 2>&1); then + log 2 "error retrieving UploadId: $parts_upload_id" + return 1 + fi + if [ "$parts_upload_id" != "$3" ]; then + log 2 "expected '$3', UploadId value is '$parts_upload_id'" + return 1 + fi + if ! part_count=$(xmllint --xpath 'count(//*[local-name()="Part"])' "$TEST_FILE_FOLDER/parts.txt" 2>&1); then + log 2 "error retrieving part count: $part_count" + return 1 + fi + if [ "$part_count" != "$4" ]; then + log 2 "expected $4, 'Part' count is '$part_count'" + return 1 + fi + if [ "$4" == 0 ]; then + return 0 + fi + if ! etags=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/parts.txt" | tr '\n' ' ' 2>&1); then + log 2 "error retrieving etags: $etags" + return 1 + fi + read -ra etags_array <<< "$etags" + shift 4 + idx=0 + while [ $# -gt 0 ]; do + if [ "$1" != "${etags_array[$idx]}" ]; then + log 2 "etag mismatch (expected '$1', actual ${etags_array[$idx]})" + return 1 + fi + ((idx++)) + shift + done + return 0 +} + +upload_check_parts() { + if [ $# -ne 6 ]; then + log 2 "'upload_check_parts' requires bucket, key, part list" + return 1 + fi + if ! create_upload_and_get_id_rest "$1" "$2"; then + log 2 "error creating upload" + return 1 + fi + # shellcheck disable=SC2154 + if ! check_part_list_rest "$1" "$2" "$upload_id" 0; then + log 2 "error checking part list before part upload" + return 1 + fi + parts_payload="" + if ! upload_check_part "$1" "$2" "$upload_id" 1 "$3"; then + log 2 "error uploading and checking first part" + return 1 + fi + # shellcheck disable=SC2154 + etag_one=$etag + if ! upload_check_part "$1" "$2" "$upload_id" 2 "$4" "$etag_one"; then + log 2 "error uploading and checking second part" + return 1 + fi + etag_two=$etag + if ! upload_check_part "$1" "$2" "$upload_id" 3 "$5" "$etag_one" "$etag_two"; then + log 2 "error uploading and checking third part" + return 1 + fi + etag_three=$etag + if ! upload_check_part "$1" "$2" "$upload_id" 4 "$6" "$etag_one" "$etag_two" "$etag_three"; then + log 2 "error uploading and checking fourth part" + return 1 + fi + log 5 "PARTS PAYLOAD: $parts_payload" + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then + log 2 "error completing multipart upload: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")" + return 1 + fi + return 0 +} + +upload_check_part() { + if [ $# -lt 5 ]; then + log 2 "'upload_check_part' requires bucket, key, upload ID, part number, part, etags" + return 1 + fi + if ! upload_part_and_get_etag_rest "$1" "$2" "$3" "$4" "$5"; then + log 2 "error uploading part $4" + return 1 + fi + parts_payload+="$etag$4" + # shellcheck disable=SC2068 + if ! check_part_list_rest "$1" "$2" "$3" "$4" "${@:6}" "$etag"; then + log 2 "error checking part list after upload $4" + return 1 + fi +} diff --git a/tests/util_multipart.sh b/tests/util_multipart.sh index 8908fc1..c247d50 100644 --- a/tests/util_multipart.sh +++ b/tests/util_multipart.sh @@ -218,36 +218,32 @@ create_and_list_multipart_uploads() { multipart_upload_from_bucket() { if [ $# -ne 4 ]; then - echo "multipart upload from bucket command missing bucket, copy source, key, and/or part count" + log 2 "multipart upload from bucket command missing bucket, copy source, key, and/or part count" return 1 fi - split_file "$3" "$4" || split_result=$? - if [[ $split_result -ne 0 ]]; then - echo "error splitting file" + if ! split_file "$3" "$4"; then + log 2 "error splitting file" return 1 fi for ((i=0;i<$4;i++)) { echo "key: $3" - put_object "s3api" "$3-$i" "$1" "$2-$i" || copy_result=$? - if [[ $copy_result -ne 0 ]]; then - echo "error copying object" + if ! put_object "s3api" "$3-$i" "$1" "$2-$i"; then + log 2 "error copying object" return 1 fi } - create_multipart_upload "$1" "$2-copy" || upload_result=$? - if [[ $upload_result -ne 0 ]]; then - echo "error running first multpart upload" + if ! create_multipart_upload "$1" "$2-copy"; then + log 2 "error running first multpart upload" return 1 fi parts="[" for ((i = 1; i <= $4; i++)); do - upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i" || local upload_result=$? - if [[ $upload_result -ne 0 ]]; then - echo "error uploading part $i" + if ! upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i"; then + log 2 "error uploading part $i" return 1 fi parts+="{\"ETag\": $etag, \"PartNumber\": $i}" @@ -257,9 +253,8 @@ multipart_upload_from_bucket() { done parts+="]" - error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}') || local completed=$? - if [[ $completed -ne 0 ]]; then - echo "Error completing upload: $error" + if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}' 2>&1); then + log 2 "Error completing upload: $error" return 1 fi return 0 @@ -580,18 +575,34 @@ create_abort_multipart_upload_rest() { log 2 "'create_abort_upload_rest' requires bucket, key" return 1 fi + if ! list_and_check_upload "$1" "$2"; then + log 2 "error listing multipart uploads before creation" + return 1 + fi + log 5 "uploads before upload: $(cat "$TEST_FILE_FOLDER/uploads.txt")" if ! create_upload_and_get_id_rest "$1" "$2"; then log 2 "error creating upload" return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" ./tests/rest_scripts/abort_multipart_upload.sh); then + if ! list_and_check_upload "$1" "$2" "$upload_id"; then + log 2 "error listing multipart uploads after upload creation" + return 1 + fi + log 5 "uploads after upload creation: $(cat "$TEST_FILE_FOLDER/uploads.txt")" + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/abort_multipart_upload.sh); then log 2 "error aborting multipart upload: $result" return 1 fi if [ "$result" != "204" ]; then - log 2 "expected '204' response, actual was '$result' (error: $(cat "$TEST_FILE_FOLDER"/output.txt)" + log 2 "expected '204' response, actual was '$result' (error: $(cat "$TEST_FILE_FOLDER"/result.txt)" return 1 fi + log 5 "final uploads: $(cat "$TEST_FILE_FOLDER/uploads.txt")" + if ! list_and_check_upload "$1" "$2"; then + log 2 "error listing multipart uploads after abort" + return 1 + fi + return 0 } multipart_upload_range_too_large() { @@ -610,3 +621,45 @@ multipart_upload_range_too_large() { fi return 0 } + +list_and_check_upload() { + if [ $# -lt 2 ]; then + log 2 "'list_and_check_upload' requires bucket, key, upload ID (optional)" + return 1 + fi + if ! uploads=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/uploads.txt" ./tests/rest_scripts/list_multipart_uploads.sh); then + log 2 "error listing multipart uploads before upload: $result" + return 1 + fi + if ! upload_count=$(xmllint --xpath 'count(//*[local-name()="Upload"])' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then + log 2 "error retrieving upload count: $upload_count" + return 1 + fi + if [[ (( $# == 2 ) && ( $upload_count != 0 )) ]]; then + log 2 "upload count mismatch (expected 0, actual $upload_count)" + return 1 + elif [[ (( $# == 3 ) && ( $upload_count != 1 )) ]]; then + log 2 "upload count mismatch (expected 1, actual $upload_count)" + return 1 + fi + if [ $# -eq 2 ]; then + return 0 + fi + if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then + log 2 "error retrieving key: $key" + return 1 + fi + if [ "$key" != "$2" ]; then + log 2 "key mismatch (expected '$2', actual '$key')" + return 1 + fi + if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then + log 2 "error retrieving upload ID: $upload_id" + return 1 + fi + if [ "$upload_id" != "$3" ]; then + log 2 "upload ID mismatch (expected '$3', actual '$upload_id')" + return 1 + fi + return 0 +}