From 94fc70f5a77c2931b2b649e7548690d638853281 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Wed, 4 Dec 2024 12:34:22 -0300 Subject: [PATCH] test: GetBucketAcl, cleanup --- tests/rest_scripts/get_bucket_acl.sh | 43 ++++++++++ tests/test_rest.sh | 14 +++- tests/test_s3api_multipart.sh | 1 + tests/test_s3api_policy.sh | 1 + tests/test_s3api_policy_multipart.sh | 2 + tests/util/util_acl.sh | 44 ++++++++++ tests/util/util_bucket.sh | 2 + tests/util/util_multipart.sh | 114 -------------------------- tests/util/util_multipart_abort.sh | 115 +++++++++++++++++++++++++++ tests/util/util_policy.sh | 1 - tests/util/util_versioning.sh | 38 ++++++--- 11 files changed, 245 insertions(+), 130 deletions(-) create mode 100755 tests/rest_scripts/get_bucket_acl.sh create mode 100644 tests/util/util_multipart_abort.sh diff --git a/tests/rest_scripts/get_bucket_acl.sh b/tests/rest_scripts/get_bucket_acl.sh new file mode 100755 index 00000000..1b3825c6 --- /dev/null +++ b/tests/rest_scripts/get_bucket_acl.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="GET +/$bucket_name +acl= +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?acl=" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/test_rest.sh b/tests/test_rest.sh index a27e9e29..0c312e38 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -29,6 +29,7 @@ source ./tests/commands/put_object_tagging.sh source ./tests/logger.sh source ./tests/setup.sh source ./tests/util/util.sh +source ./tests/util/util_acl.sh source ./tests/util/util_attributes.sh source ./tests/util/util_legal_hold.sh source ./tests/util/util_list_buckets.sh @@ -189,7 +190,6 @@ export RUN_USERS=true } @test "test_rest_versioning" { - skip "https://github.com/versity/versitygw/issues/864" test_file="test_file" run setup_bucket "s3api" "$BUCKET_ONE_NAME" @@ -222,7 +222,6 @@ export RUN_USERS=true } @test "versioning - add version, then delete and check for marker" { - skip "https://github.com/versity/versitygw/issues/864" test_file="test_file" run setup_bucket "s3api" "$BUCKET_ONE_NAME" @@ -442,4 +441,15 @@ export RUN_USERS=true run put_and_check_policy_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/policy_file.txt" "Allow" "$USERNAME_ONE" "s3:PutBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME" assert_success +} + +@test "REST - get ACL" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/971" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run get_and_check_acl_rest "$BUCKET_ONE_NAME" + assert_success } \ No newline at end of file diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index c5020235..9ae6c7d9 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -18,6 +18,7 @@ source ./tests/setup.sh source ./tests/test_s3api_root_inner.sh source ./tests/util/util_file.sh source ./tests/util/util_multipart.sh +source ./tests/util/util_multipart_abort.sh source ./tests/util/util_tags.sh source ./tests/commands/get_object.sh source ./tests/commands/put_object.sh diff --git a/tests/test_s3api_policy.sh b/tests/test_s3api_policy.sh index a7df18ed..7fa5bb03 100755 --- a/tests/test_s3api_policy.sh +++ b/tests/test_s3api_policy.sh @@ -20,6 +20,7 @@ source ./tests/test_s3api_policy_bucket.sh source ./tests/test_s3api_policy_multipart.sh source ./tests/test_s3api_policy_object.sh source ./tests/util/util_multipart.sh +source ./tests/util/util_multipart_abort.sh source ./tests/util/util_file.sh source ./tests/util/util_policy.sh source ./tests/util/util_tags.sh diff --git a/tests/test_s3api_policy_multipart.sh b/tests/test_s3api_policy_multipart.sh index 72252596..d8864387 100644 --- a/tests/test_s3api_policy_multipart.sh +++ b/tests/test_s3api_policy_multipart.sh @@ -14,6 +14,8 @@ # specific language governing permissions and limitations # under the License. +source ./tests/util/util_multipart_abort.sh + test_s3api_policy_abort_multipart_upload() { policy_file="policy_file" test_file="test_file" diff --git a/tests/util/util_acl.sh b/tests/util/util_acl.sh index 42c4acfb..40090572 100644 --- a/tests/util/util_acl.sh +++ b/tests/util/util_acl.sh @@ -199,3 +199,47 @@ get_check_acl_after_policy() { fi fi } + +get_and_check_acl_rest() { + if [ $# -ne 1 ]; then + log 2 "'get_and_check_acl_rest' requires bucket name" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/acl.txt" ./tests/rest_scripts/get_bucket_acl.sh); then + log 2 "error attempting to get bucket ACL response: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "get acl returned code '$result' (message: $(cat "$TEST_FILE_FOLDER/acl.txt"))" + return 1 + fi + log 5 "acl: $(cat "$TEST_FILE_FOLDER/acl.txt")" + if ! access_control_policy=$(xmllint --xpath '//*[local-name()="AccessControlPolicy"]' "$TEST_FILE_FOLDER/acl.txt" 2>&1); then + log 2 "error getting access control policy: $access_control_policy" + return 1 + fi + if ! owner=$(echo "$access_control_policy" | xmllint --xpath '//*[local-name()="Owner"]' - 2>&1); then + log 2 "error getting owner information: $owner" + return 1 + fi + if [ "$DIRECT" == "true" ]; then + if ! display_name=$(echo "$owner" | xmllint --xpath '//*[local-name()="DisplayName"]/text()' - 2>&1); then + log 2 "error getting display name: $display_name" + return 1 + fi + if [ "$display_name" != "$DIRECT_DISPLAY_NAME" ]; then + log 2 "display name mismatch (expected '$DIRECT_DISPLAY_NAME', actual '$display_name')" + return 1 + fi + else + if ! id=$(echo "$owner" | xmllint --xpath '//*[local-name()="ID"]/text()' - 2>&1); then + log 2 "error getting ID: $id" + return 1 + fi + if [ "$id" != "$AWS_ACCESS_KEY_ID" ]; then + log 2 "ID mismatch" + return 1 + fi + fi + return 0 +} diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index ebc52080..c183dbf9 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +source ./tests/util/util_multipart_abort.sh + # recursively delete an AWS bucket # param: client, bucket name # fail if error diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh index dcf87fd6..cb8dcf5c 100644 --- a/tests/util/util_multipart.sh +++ b/tests/util/util_multipart.sh @@ -476,68 +476,6 @@ multipart_upload_with_params() { return 0 } -# run upload, then abort it -# params: bucket, key, local file location, number of parts to split into before uploading -# return 0 for success, 1 for failure -run_then_abort_multipart_upload() { - if [ $# -ne 4 ]; then - log 2 "run then abort multipart upload command missing bucket, key, file, and/or part count" - return 1 - fi - - if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then - log 2 "error performing pre-completion multipart upload" - return 1 - fi - - if ! abort_multipart_upload "$1" "$2" "$upload_id"; then - log 2 "error aborting multipart upload" - return 1 - fi - return 0 -} - -# param: bucket name -# return 0 for success, 1 for error -abort_all_multipart_uploads() { - if [ $# -ne 1 ]; then - log 2 "'abort_all_multipart_uploads' requires bucket name" - return 1 - fi - if ! list_multipart_uploads "$1"; then - log 2 "error listing multipart uploads" - return 1 - fi - # shellcheck disable=SC2154 - log 5 "UPLOADS: $uploads" - if ! upload_set=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -c '.Uploads[]' 2>&1); then - if [[ $upload_set == *"Cannot iterate over null"* ]]; then - return 0 - else - log 2 "error getting upload set: $upload_set" - return 1 - fi - fi - log 5 "UPLOAD SET: $upload_set" - for upload in $upload_set; do - log 5 "UPLOAD: $upload" - if ! upload_id=$(echo "$upload" | jq -r ".UploadId" 2>&1); then - log 2 "error getting upload ID: $upload_id" - return 1 - fi - log 5 "upload ID: $upload_id" - if ! key=$(echo "$upload" | jq -r ".Key" 2>&1); then - log 2 "error getting key: $key" - return 1 - fi - log 5 "Aborting multipart upload for key: $key, UploadId: $upload_id" - if ! abort_multipart_upload "$1" "$key" "$upload_id"; then - log 2 "error aborting multipart upload" - return 1 - fi - done -} - create_upload_and_get_id_rest() { if [ $# -ne 2 ]; then log 2 "'create_upload_and_get_id_rest' requires bucket, key" @@ -560,41 +498,6 @@ create_upload_and_get_id_rest() { return 0 } -create_abort_multipart_upload_rest() { - if [ $# -ne 2 ]; then - log 2 "'create_abort_upload_rest' requires bucket, key" - return 1 - fi - if ! list_and_check_upload "$1" "$2"; then - log 2 "error listing multipart uploads before creation" - return 1 - fi - log 5 "uploads before upload: $(cat "$TEST_FILE_FOLDER/uploads.txt")" - if ! create_upload_and_get_id_rest "$1" "$2"; then - log 2 "error creating upload" - return 1 - fi - if ! list_and_check_upload "$1" "$2" "$upload_id"; then - log 2 "error listing multipart uploads after upload creation" - return 1 - fi - log 5 "uploads after upload creation: $(cat "$TEST_FILE_FOLDER/uploads.txt")" - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/abort_multipart_upload.sh); then - log 2 "error aborting multipart upload: $result" - return 1 - fi - if [ "$result" != "204" ]; then - log 2 "expected '204' response, actual was '$result' (error: $(cat "$TEST_FILE_FOLDER"/result.txt)" - return 1 - fi - log 5 "final uploads: $(cat "$TEST_FILE_FOLDER/uploads.txt")" - if ! list_and_check_upload "$1" "$2"; then - log 2 "error listing multipart uploads after abort" - return 1 - fi - return 0 -} - multipart_upload_range_too_large() { if [ $# -ne 3 ]; then log 2 "'multipart_upload_range_too_large' requires bucket name, key, file location" @@ -680,23 +583,6 @@ run_and_verify_multipart_upload_with_valid_range() { return 0 } -check_abort_access_denied() { - if [ $# -ne 5 ]; then - log 2 "'check_abort_access_denied' requires bucket, file, username, password" - return 1 - fi - if abort_multipart_upload_with_user "$1" "$2" "$3" "$4" "$5"; then - log 2 "abort multipart upload succeeded despite lack of permissions" - return 1 - fi - # shellcheck disable=SC2154 - if [[ "$abort_multipart_upload_error" != *"AccessDenied"* ]]; then - log 2 "unexpected abort error: $abort_multipart_upload_error" - return 1 - fi - return 0 -} - list_check_multipart_upload_key() { if [ $# -ne 4 ]; then log 2 "'list_check_multipart_upload_key' requires bucket, username, password, expected key" diff --git a/tests/util/util_multipart_abort.sh b/tests/util/util_multipart_abort.sh new file mode 100644 index 00000000..fb4a2a58 --- /dev/null +++ b/tests/util/util_multipart_abort.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +check_abort_access_denied() { + if [ $# -ne 5 ]; then + log 2 "'check_abort_access_denied' requires bucket, file, username, password" + return 1 + fi + if abort_multipart_upload_with_user "$1" "$2" "$3" "$4" "$5"; then + log 2 "abort multipart upload succeeded despite lack of permissions" + return 1 + fi + # shellcheck disable=SC2154 + if [[ "$abort_multipart_upload_error" != *"AccessDenied"* ]]; then + log 2 "unexpected abort error: $abort_multipart_upload_error" + return 1 + fi + return 0 +} + +create_abort_multipart_upload_rest() { + if [ $# -ne 2 ]; then + log 2 "'create_abort_upload_rest' requires bucket, key" + return 1 + fi + if ! list_and_check_upload "$1" "$2"; then + log 2 "error listing multipart uploads before creation" + return 1 + fi + log 5 "uploads before upload: $(cat "$TEST_FILE_FOLDER/uploads.txt")" + if ! create_upload_and_get_id_rest "$1" "$2"; then + log 2 "error creating upload" + return 1 + fi + if ! list_and_check_upload "$1" "$2" "$upload_id"; then + log 2 "error listing multipart uploads after upload creation" + return 1 + fi + log 5 "uploads after upload creation: $(cat "$TEST_FILE_FOLDER/uploads.txt")" + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/abort_multipart_upload.sh); then + log 2 "error aborting multipart upload: $result" + return 1 + fi + if [ "$result" != "204" ]; then + log 2 "expected '204' response, actual was '$result' (error: $(cat "$TEST_FILE_FOLDER"/result.txt)" + return 1 + fi + log 5 "final uploads: $(cat "$TEST_FILE_FOLDER/uploads.txt")" + if ! list_and_check_upload "$1" "$2"; then + log 2 "error listing multipart uploads after abort" + return 1 + fi + return 0 +} + +# param: bucket name +# return 0 for success, 1 for error +abort_all_multipart_uploads() { + if [ $# -ne 1 ]; then + log 2 "'abort_all_multipart_uploads' requires bucket name" + return 1 + fi + if ! list_multipart_uploads "$1"; then + log 2 "error listing multipart uploads" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "UPLOADS: $uploads" + if ! upload_set=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -c '.Uploads[]' 2>&1); then + if [[ $upload_set == *"Cannot iterate over null"* ]]; then + return 0 + else + log 2 "error getting upload set: $upload_set" + return 1 + fi + fi + log 5 "UPLOAD SET: $upload_set" + for upload in $upload_set; do + log 5 "UPLOAD: $upload" + if ! upload_id=$(echo "$upload" | jq -r ".UploadId" 2>&1); then + log 2 "error getting upload ID: $upload_id" + return 1 + fi + log 5 "upload ID: $upload_id" + if ! key=$(echo "$upload" | jq -r ".Key" 2>&1); then + log 2 "error getting key: $key" + return 1 + fi + log 5 "Aborting multipart upload for key: $key, UploadId: $upload_id" + if ! abort_multipart_upload "$1" "$key" "$upload_id"; then + log 2 "error aborting multipart upload" + return 1 + fi + done +} + +# run upload, then abort it +# params: bucket, key, local file location, number of parts to split into before uploading +# return 0 for success, 1 for failure +run_then_abort_multipart_upload() { + if [ $# -ne 4 ]; then + log 2 "run then abort multipart upload command missing bucket, key, file, and/or part count" + return 1 + fi + + if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then + log 2 "error performing pre-completion multipart upload" + return 1 + fi + + if ! abort_multipart_upload "$1" "$2" "$upload_id"; then + log 2 "error aborting multipart upload" + return 1 + fi + return 0 +} diff --git a/tests/util/util_policy.sh b/tests/util/util_policy.sh index af1130a3..06455091 100644 --- a/tests/util/util_policy.sh +++ b/tests/util/util_policy.sh @@ -310,7 +310,6 @@ put_and_check_policy_rest() { log 2 "unexpected response code, expected '200' or '204', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/result.txt"))" return 1 fi - log 5 "response: $(cat "$TEST_FILE_FOLDER/result.txt")" if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then log 2 "error attempting to get bucket policy response: $result" return 1 diff --git a/tests/util/util_versioning.sh b/tests/util/util_versioning.sh index acbe2145..244b8254 100644 --- a/tests/util/util_versioning.sh +++ b/tests/util/util_versioning.sh @@ -49,23 +49,35 @@ delete_old_versions() { log 5 "version keys: ${version_keys[*]}" log 5 "version IDs: ${version_ids[*]}" for idx in "${!version_keys[@]}"; do - log 5 "idx: $idx" - log 5 "version ID: ${version_ids[$idx]}" - # shellcheck disable=SC2154 - if [ "$lock_config_exists" == "true" ]; then - if ! delete_object_version_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then - log 2 "error deleting object version" - return 1 - fi - else - if ! delete_object_version "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then - log 2 "error deleting object version" - return 1 - fi + if ! delete_object_version_with_or_without_retention "$1"; then + log 2 "error deleting version with or without retention" + return 1 fi done } +delete_object_version_with_or_without_retention() { + if [ $# -ne 1 ]; then + log 2 "'delete_object_version_with_or_without_retention' requires bucket name" + return 1 + fi + log 5 "idx: $idx" + log 5 "version ID: ${version_ids[$idx]}" + # shellcheck disable=SC2154 + if [ "$lock_config_exists" == "true" ]; then + if ! delete_object_version_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + log 2 "error deleting object version" + return 1 + fi + else + if ! delete_object_version "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + log 2 "error deleting object version" + return 1 + fi + fi + return 0 +} + parse_version_data() { if [ $# -ne 2 ]; then log 2 "'parse_version_data' requires raw data, element name"