From cc7ea685d90e8d38de443ffef8eec11123afa820 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Mon, 6 Jan 2025 15:04:12 -0300 Subject: [PATCH] test: REST HeadObject --- tests/remove_static.sh | 2 +- tests/rest_scripts/head_object.sh | 47 +++++++ tests/setup.sh | 4 +- tests/setup_static.sh | 2 +- tests/teardown_static.sh | 2 +- tests/test_common.sh | 2 +- tests/test_common_acl.sh | 1 - tests/test_rest.sh | 25 +++- tests/test_rest_acl.sh | 2 +- tests/test_s3api_bucket.sh | 2 +- tests/test_s3api_multipart.sh | 23 +--- tests/test_s3api_object.sh | 2 +- tests/test_s3cmd.sh | 2 +- tests/test_user_common.sh | 2 +- tests/util/util_acl.sh | 57 +++++++++ tests/util/util_attributes.sh | 17 +++ tests/util/util_aws.sh | 16 ++- tests/util/util_bucket.sh | 3 + tests/util/util_head_object.sh | 18 +++ tests/util/util_multipart.sh | 40 ++++++ tests/util/{util.sh => util_object.sh} | 163 ------------------------- tests/util/util_policy.sh | 13 ++ tests/util/util_retention.sh | 94 ++++++++++++++ 23 files changed, 342 insertions(+), 197 deletions(-) create mode 100755 tests/rest_scripts/head_object.sh rename tests/util/{util.sh => util_object.sh} (66%) create mode 100644 tests/util/util_retention.sh diff --git a/tests/remove_static.sh b/tests/remove_static.sh index 78d1f60..d459261 100755 --- a/tests/remove_static.sh +++ b/tests/remove_static.sh @@ -15,7 +15,7 @@ # under the License. source ./tests/setup.sh -source ./tests/util/util.sh +source ./tests/util/util_object.sh delete_bucket_if_exists() { if [[ $# -ne 2 ]]; then diff --git a/tests/rest_scripts/head_object.sh b/tests/rest_scripts/head_object.sh new file mode 100755 index 0000000..6e0a419 --- /dev/null +++ b/tests/rest_scripts/head_object.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2154 +key="$OBJECT_KEY" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +#x-amz-object-attributes:ETag +canonical_request="HEAD +/$bucket_name/$key + +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ksI -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/setup.sh b/tests/setup.sh index 8b61f22..5f688a7 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -17,7 +17,7 @@ source ./tests/env.sh source ./tests/report.sh source ./tests/setup_mc.sh -source ./tests/util/util.sh +source ./tests/util/util_object.sh source ./tests/versity.sh # bats setup function @@ -50,6 +50,7 @@ setup() { fi export AWS_PROFILE + log 4 "********** END SETUP **********" } delete_temp_log_if_exists() { @@ -65,6 +66,7 @@ delete_temp_log_if_exists() { # bats teardown function teardown() { # shellcheck disable=SC2154 + log 4 "********** BEGIN TEARDOWN **********" if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ]; then log 5 "deleting or clearing buckets" if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then diff --git a/tests/setup_static.sh b/tests/setup_static.sh index 64f6501..0bd2483 100755 --- a/tests/setup_static.sh +++ b/tests/setup_static.sh @@ -15,7 +15,7 @@ # under the License. source ./tests/env.sh -source ./tests/util/util.sh +source ./tests/util/util_object.sh source ./tests/commands/create_bucket.sh create_bucket_if_not_exists() { diff --git a/tests/teardown_static.sh b/tests/teardown_static.sh index db0e46e..569253a 100755 --- a/tests/teardown_static.sh +++ b/tests/teardown_static.sh @@ -15,7 +15,7 @@ # under the License. source ./tests/setup.sh -source ./tests/util/util.sh +source ./tests/util/util_object.sh if ! base_setup; then log 2 "error starting versity to set up static buckets" diff --git a/tests/test_common.sh b/tests/test_common.sh index b03d7f0..3329275 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -15,11 +15,11 @@ # under the License. source ./tests/setup.sh -source ./tests/util/util.sh source ./tests/util/util_acl.sh source ./tests/util/util_bucket_location.sh source ./tests/util/util_file.sh source ./tests/util/util_list_buckets.sh +source ./tests/util/util_object.sh source ./tests/util/util_policy.sh source ./tests/util/util_presigned_url.sh source ./tests/commands/copy_object.sh diff --git a/tests/test_common_acl.sh b/tests/test_common_acl.sh index 7d05bc3..e017888 100644 --- a/tests/test_common_acl.sh +++ b/tests/test_common_acl.sh @@ -73,7 +73,6 @@ test_common_put_bucket_acl() { assert_success log 5 "acl: $(cat "$TEST_FILE_FOLDER/$acl_file")" - run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file" assert_success diff --git a/tests/test_rest.sh b/tests/test_rest.sh index d8aaebf..c66f1d2 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -31,15 +31,16 @@ source ./tests/commands/put_object_retention.sh source ./tests/commands/put_object_tagging.sh source ./tests/logger.sh source ./tests/setup.sh -source ./tests/util/util.sh source ./tests/util/util_acl.sh source ./tests/util/util_attributes.sh +source ./tests/util/util_head_object.sh source ./tests/util/util_legal_hold.sh source ./tests/util/util_list_buckets.sh source ./tests/util/util_list_objects.sh source ./tests/util/util_list_parts.sh source ./tests/util/util_lock_config.sh source ./tests/util/util_multipart_before_completion.sh +source ./tests/util/util_object.sh source ./tests/util/util_ownership.sh source ./tests/util/util_policy.sh source ./tests/util/util_public_access_block.sh @@ -522,3 +523,25 @@ export RUN_USERS=true run download_and_compare_file "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" assert_success } + +@test "REST - head object" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1018" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test_file" + run create_test_file "$test_file" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run get_etag_rest "$BUCKET_ONE_NAME" "$test_file" + assert_success + expected_etag=$output + + run get_etag_attribute_rest "$BUCKET_ONE_NAME" "$test_file" "$expected_etag" + assert_success +} diff --git a/tests/test_rest_acl.sh b/tests/test_rest_acl.sh index 954bebc..c910688 100755 --- a/tests/test_rest_acl.sh +++ b/tests/test_rest_acl.sh @@ -20,8 +20,8 @@ load ./bats-assert/load source ./tests/commands/put_object.sh source ./tests/logger.sh source ./tests/setup.sh -source ./tests/util/util.sh source ./tests/util/util_acl.sh +source ./tests/util/util_object.sh export RUN_USERS=true diff --git a/tests/test_s3api_bucket.sh b/tests/test_s3api_bucket.sh index fb2da08..d966f84 100755 --- a/tests/test_s3api_bucket.sh +++ b/tests/test_s3api_bucket.sh @@ -18,11 +18,11 @@ load ./bats-support/load load ./bats-assert/load source ./tests/setup.sh -source ./tests/util/util.sh source ./tests/util/util_create_bucket.sh source ./tests/util/util_file.sh source ./tests/util/util_head_bucket.sh source ./tests/util/util_lock_config.sh +source ./tests/util/util_object.sh source ./tests/util/util_tags.sh source ./tests/util/util_users.sh source ./tests/test_s3api_root_inner.sh diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index fb382e4..e5e10de 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -79,28 +79,9 @@ source ./tests/commands/list_multipart_uploads.sh local expected_tag_key="TestTag" local expected_tag_val="TestTagVal" - os_name="$(uname)" - if [[ "$os_name" == "Darwin" ]]; then - now=$(date -u +"%Y-%m-%dT%H:%M:%S") - later=$(date -j -v +15S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S") - else - now=$(date +"%Y-%m-%dT%H:%M:%S") - later=$(date -d "$now 15 seconds" +"%Y-%m-%dT%H:%M:%S") - fi - - run create_test_files "$bucket_file" + run setup_multipart_upload_with_params "$BUCKET_ONE_NAME" "$bucket_file" assert_success - - run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1 - assert_success - - run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME" - assert_success - # in static bucket config, bucket will still exist - if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then - run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" - assert_success - fi + later=${lines[${#lines[@]}-1]} run multipart_upload_with_params "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 \ "$expected_content_type" \ diff --git a/tests/test_s3api_object.sh b/tests/test_s3api_object.sh index 11c4322..ae78a2a 100755 --- a/tests/test_s3api_object.sh +++ b/tests/test_s3api_object.sh @@ -18,10 +18,10 @@ load ./bats-support/load load ./bats-assert/load source ./tests/setup.sh -source ./tests/util/util.sh source ./tests/util/util_create_bucket.sh source ./tests/util/util_file.sh source ./tests/util/util_lock_config.sh +source ./tests/util/util_object.sh source ./tests/util/util_tags.sh source ./tests/util/util_users.sh source ./tests/test_s3api_root_inner.sh diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 88d1d55..3199841 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -20,8 +20,8 @@ load ./bats-assert/load source ./tests/setup.sh source ./tests/test_common.sh source ./tests/test_common_acl.sh -source ./tests/util/util.sh source ./tests/util/util_create_bucket.sh +source ./tests/util/util_object.sh source ./tests/util/util_users.sh source ./tests/commands/delete_bucket_policy.sh source ./tests/commands/get_bucket_policy.sh diff --git a/tests/test_user_common.sh b/tests/test_user_common.sh index 04d9b6a..9e494df 100755 --- a/tests/test_user_common.sh +++ b/tests/test_user_common.sh @@ -15,9 +15,9 @@ # under the License. source ./tests/setup.sh -source ./tests/util/util.sh source ./tests/util/util_create_bucket.sh source ./tests/util/util_list_buckets.sh +source ./tests/util/util_object.sh source ./tests/util/util_users.sh source ./tests/commands/list_buckets.sh diff --git a/tests/util/util_acl.sh b/tests/util/util_acl.sh index f346059..8902ee7 100644 --- a/tests/util/util_acl.sh +++ b/tests/util/util_acl.sh @@ -388,3 +388,60 @@ put_canned_acl_rest() { fi return 0 } + +# param: bucket name +# return 0 for success, 1 for failure +check_ownership_rule_and_reset_acl() { + if [ $# -ne 1 ]; then + log 2 "'check_ownership_rule_and_reset_acl' requires bucket name" + return 1 + fi + if ! get_bucket_ownership_controls "$1"; then + log 2 "error getting bucket ownership controls" + return 1 + fi + # shellcheck disable=SC2154 + if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then + log 2 "error getting object ownership rule: $object_ownership_rule" + return 1 + fi + if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then + log 2 "error resetting bucket ACL" + return 1 + fi +} + +# param: bucket name +# return 1 for failure, 0 for success +get_object_ownership_rule_and_update_acl() { + if [ $# -ne 1 ]; then + log 2 "'get_object_ownership_rule_and_update_acl' requires bucket name" + return 1 + fi + if ! get_object_ownership_rule "$1"; then + log 2 "error getting object ownership rule" + return 1 + fi + log 5 "object ownership rule: $object_ownership_rule" + if [[ "$object_ownership_rule" != "BucketOwnerEnforced" ]] && ! put_bucket_canned_acl "$1" "private"; then + log 2 "error resetting bucket ACLs" + return 1 + fi +} + +# get object acl +# param: object path +# export acl for success, return 1 for error +get_object_acl() { + if [ $# -ne 2 ]; then + log 2 "object ACL command missing object name" + return 1 + fi + local exit_code=0 + acl=$(aws --no-verify-ssl s3api get-object-acl --bucket "$1" --key "$2" 2>&1) || exit_code="$?" + if [ $exit_code -ne 0 ]; then + log 2 "Error getting object ACLs: $acl" + return 1 + fi + export acl +} diff --git a/tests/util/util_attributes.sh b/tests/util/util_attributes.sh index e945669..d909b4d 100644 --- a/tests/util/util_attributes.sh +++ b/tests/util/util_attributes.sh @@ -112,4 +112,21 @@ add_and_check_checksum() { log 2 "empty checksum" return 1 fi +} + +get_etag_attribute_rest() { + if [ $# -ne 3 ]; then + log 2 "'get_etag_attribute_rest' requires bucket name, object key, expected etag" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" ATTRIBUTES="ETag" OUTPUT_FILE="$TEST_FILE_FOLDER/attributes.txt" ./tests/rest_scripts/get_object_attributes.sh); then + log 2 "error attempting to get object info: $result" + return 1 + fi + log 5 "attributes: $(cat "$TEST_FILE_FOLDER/attributes.txt")" + if ! check_xml_element "$TEST_FILE_FOLDER/attributes.txt" "$3" "GetObjectAttributesResponse" "ETag"; then + log 2 "etag mismatch" + return 1 + fi + return 0 } \ No newline at end of file diff --git a/tests/util/util_aws.sh b/tests/util/util_aws.sh index dcae4e9..212d6f0 100644 --- a/tests/util/util_aws.sh +++ b/tests/util/util_aws.sh @@ -51,4 +51,18 @@ abort_all_multipart_uploads() { fi done <<< "$lines" return 0 -} \ No newline at end of file +} + +remove_insecure_request_warning() { + if [[ $# -ne 1 ]]; then + log 2 "remove insecure request warning requires input lines" + return 1 + fi + parsed_output=() + while IFS= read -r line; do + if [[ $line != *InsecureRequestWarning* ]]; then + parsed_output+=("$line") + fi + done <<< "$1" + export parsed_output +} diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index 5c992d7..580fb6c 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash +source ./tests/util/util_acl.sh source ./tests/util/util_multipart_abort.sh +source ./tests/util/util_policy.sh +source ./tests/util/util_retention.sh # recursively delete an AWS bucket # param: client, bucket name diff --git a/tests/util/util_head_object.sh b/tests/util/util_head_object.sh index 53d8f70..82e6dcf 100644 --- a/tests/util/util_head_object.sh +++ b/tests/util/util_head_object.sh @@ -69,4 +69,22 @@ get_and_verify_metadata() { return 1 fi return 0 +} + +get_etag_rest() { + if [ $# -ne 2 ]; then + log 2 "'get_etag_rest' requires bucket name, object key" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/head_object.txt" ./tests/rest_scripts/head_object.sh); then + log 2 "error attempting to get object info: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "response code '$result', data: $(cat "$TEST_FILE_FOLDER/head_object.txt")" + return 1 + fi + log 5 "head object data: $(cat "$TEST_FILE_FOLDER/head_object.txt")" + etag_value=$(grep "E[Tt]ag:" "$TEST_FILE_FOLDER/head_object.txt" | sed -n 's/E[Tt]ag: "\([^"]*\)"/\1/p' | tr -d '\r') + echo "$etag_value" } \ No newline at end of file diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh index c30a25e..5a7a3f4 100644 --- a/tests/util/util_multipart.sh +++ b/tests/util/util_multipart.sh @@ -295,3 +295,43 @@ create_upload_finish_wrong_etag() { fi return 0 } + +setup_multipart_upload_with_params() { + if [ $# -ne 2 ]; then + log 2 "'setup_multipart_upload_with_params' requires bucket name, file name" + return 1 + fi + os_name="$(uname)" + if [[ "$os_name" == "Darwin" ]]; then + now=$(date -u +"%Y-%m-%dT%H:%M:%S") + later=$(date -j -v +15S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S") + else + now=$(date +"%Y-%m-%dT%H:%M:%S") + later=$(date -d "$now 15 seconds" +"%Y-%m-%dT%H:%M:%S") + fi + + if ! create_test_files "$2"; then + log 2 "error creating test file" + return 1 + fi + + if ! result=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER/$2" bs=5M count=1 2>&1); then + log 2 "error creating large file: $result" + return 1 + fi + + if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then + log 2 "error cleaning up bucket" + return 1 + fi + # in static bucket config, bucket will still exist + if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then + if ! create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"; then + log 2 "error creating bucket with object lock enabled" + return 1 + fi + fi + log 5 "later in function: $later" + echo "$later" + return 0 +} diff --git a/tests/util/util.sh b/tests/util/util_object.sh similarity index 66% rename from tests/util/util.sh rename to tests/util/util_object.sh index 4a8404d..37df50c 100644 --- a/tests/util/util.sh +++ b/tests/util/util_object.sh @@ -47,46 +47,6 @@ source ./tests/commands/upload_part_copy.sh source ./tests/commands/upload_part.sh source ./tests/util/util_users.sh -# params: bucket name -# return 0 for success, 1 for error -add_governance_bypass_policy() { - if [[ $# -ne 1 ]]; then - log 2 "'add governance bypass policy' command requires bucket name" - return 1 - fi - cat < "$TEST_FILE_FOLDER/policy-bypass-governance.txt" -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": "*", - "Action": "s3:BypassGovernanceRetention", - "Resource": "arn:aws:s3:::$1/*" - } - ] -} -EOF - if ! put_bucket_policy "s3api" "$1" "$TEST_FILE_FOLDER/policy-bypass-governance.txt"; then - log 2 "error putting governance bypass policy" - return 1 - fi - return 0 -} - -log_bucket_policy() { - if [ $# -ne 1 ]; then - log 2 "'log_bucket_policy' requires bucket name" - return - fi - if ! get_bucket_policy "s3api" "$1"; then - log 2 "error getting bucket policy" - return - fi - # shellcheck disable=SC2154 - log 5 "BUCKET POLICY: $bucket_policy" -} - # param: bucket name # return 0 for success, 1 for failure list_and_delete_objects() { @@ -114,28 +74,6 @@ list_and_delete_objects() { return 0 } -# param: bucket name -# return 0 for success, 1 for failure -check_ownership_rule_and_reset_acl() { - if [ $# -ne 1 ]; then - log 2 "'check_ownership_rule_and_reset_acl' requires bucket name" - return 1 - fi - if ! get_bucket_ownership_controls "$1"; then - log 2 "error getting bucket ownership controls" - return 1 - fi - # shellcheck disable=SC2154 - if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then - log 2 "error getting object ownership rule: $object_ownership_rule" - return 1 - fi - if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then - log 2 "error resetting bucket ACL" - return 1 - fi -} - check_object_lock_config() { if [ $# -ne 1 ]; then log 2 "'check_object_lock_config' requires bucket name" @@ -174,76 +112,6 @@ clear_object_in_bucket() { return 0 } -# params: bucket, object, possible WORM error after deletion attempt -# return 0 for success, 1 for no WORM protection, 2 for error -check_for_and_remove_worm_protection() { - if [ $# -ne 3 ]; then - log 2 "'check_for_and_remove_worm_protection' command requires bucket, object, error" - return 2 - fi - - if [[ $3 == *"WORM"* ]]; then - log 5 "WORM protection found" - if ! put_object_legal_hold "$1" "$2" "OFF"; then - log 2 "error removing object legal hold" - return 2 - fi - sleep 1 - if [[ $LOG_LEVEL_INT -ge 5 ]]; then - log_worm_protection "$1" "$2" - fi - if ! add_governance_bypass_policy "$1"; then - log 2 "error adding new governance bypass policy" - return 2 - fi - if ! delete_object_bypass_retention "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then - log 2 "error deleting object after legal hold removal" - return 2 - fi - else - log 5 "no WORM protection found" - return 1 - fi - return 0 -} - -# params: bucket name, object -log_worm_protection() { - if ! get_object_legal_hold "$1" "$2"; then - log 2 "error getting object legal hold status" - return - fi - # shellcheck disable=SC2154 - log 5 "LEGAL HOLD: $legal_hold" - if ! get_object_retention "$1" "$2"; then - log 2 "error getting object retention" - # shellcheck disable=SC2154 - if [[ $get_object_retention_error != *"NoSuchObjectLockConfiguration"* ]]; then - return - fi - fi - # shellcheck disable=SC2154 - log 5 "RETENTION: $retention" -} - -# param: bucket name -# return 1 for failure, 0 for success -get_object_ownership_rule_and_update_acl() { - if [ $# -ne 1 ]; then - log 2 "'get_object_ownership_rule_and_update_acl' requires bucket name" - return 1 - fi - if ! get_object_ownership_rule "$1"; then - log 2 "error getting object ownership rule" - return 1 - fi - log 5 "object ownership rule: $object_ownership_rule" - if [[ "$object_ownership_rule" != "BucketOwnerEnforced" ]] && ! put_bucket_canned_acl "$1" "private"; then - log 2 "error resetting bucket ACLs" - return 1 - fi -} - # check if object exists on S3 via gateway # param: command, object path # return 0 for true, 1 for false, 2 for error @@ -359,20 +227,6 @@ check_and_put_object() { return 0 } -remove_insecure_request_warning() { - if [[ $# -ne 1 ]]; then - log 2 "remove insecure request warning requires input lines" - return 1 - fi - parsed_output=() - while IFS= read -r line; do - if [[ $line != *InsecureRequestWarning* ]]; then - parsed_output+=("$line") - fi - done <<< "$1" - export parsed_output -} - # check if object info (etag) is accessible # param: path of object # return 0 for yes, 1 for no, 2 for error @@ -394,23 +248,6 @@ object_is_accessible() { return 0 } -# get object acl -# param: object path -# export acl for success, return 1 for error -get_object_acl() { - if [ $# -ne 2 ]; then - log 2 "object ACL command missing object name" - return 1 - fi - local exit_code=0 - acl=$(aws --no-verify-ssl s3api get-object-acl --bucket "$1" --key "$2" 2>&1) || exit_code="$?" - if [ $exit_code -ne 0 ]; then - log 2 "Error getting object ACLs: $acl" - return 1 - fi - export acl -} - # copy a file to/from S3 # params: source, destination # return 0 for success, 1 for failure diff --git a/tests/util/util_policy.sh b/tests/util/util_policy.sh index 0645509..9b38686 100644 --- a/tests/util/util_policy.sh +++ b/tests/util/util_policy.sh @@ -330,3 +330,16 @@ put_and_check_policy_rest() { fi return 0 } + +log_bucket_policy() { + if [ $# -ne 1 ]; then + log 2 "'log_bucket_policy' requires bucket name" + return + fi + if ! get_bucket_policy "s3api" "$1"; then + log 2 "error getting bucket policy" + return + fi + # shellcheck disable=SC2154 + log 5 "BUCKET POLICY: $bucket_policy" +} diff --git a/tests/util/util_retention.sh b/tests/util/util_retention.sh new file mode 100644 index 0000000..dec3a07 --- /dev/null +++ b/tests/util/util_retention.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# params: bucket name +# return 0 for success, 1 for error +add_governance_bypass_policy() { + if [[ $# -ne 1 ]]; then + log 2 "'add governance bypass policy' command requires bucket name" + return 1 + fi + cat < "$TEST_FILE_FOLDER/policy-bypass-governance.txt" +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:BypassGovernanceRetention", + "Resource": "arn:aws:s3:::$1/*" + } + ] +} +EOF + if ! put_bucket_policy "s3api" "$1" "$TEST_FILE_FOLDER/policy-bypass-governance.txt"; then + log 2 "error putting governance bypass policy" + return 1 + fi + return 0 +} + +# params: bucket, object, possible WORM error after deletion attempt +# return 0 for success, 1 for no WORM protection, 2 for error +check_for_and_remove_worm_protection() { + if [ $# -ne 3 ]; then + log 2 "'check_for_and_remove_worm_protection' command requires bucket, object, error" + return 2 + fi + + if [[ $3 == *"WORM"* ]]; then + log 5 "WORM protection found" + if ! put_object_legal_hold "$1" "$2" "OFF"; then + log 2 "error removing object legal hold" + return 2 + fi + sleep 1 + if [[ $LOG_LEVEL_INT -ge 5 ]]; then + log_worm_protection "$1" "$2" + fi + if ! add_governance_bypass_policy "$1"; then + log 2 "error adding new governance bypass policy" + return 2 + fi + if ! delete_object_bypass_retention "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then + log 2 "error deleting object after legal hold removal" + return 2 + fi + else + log 5 "no WORM protection found" + return 1 + fi + return 0 +} + +# params: bucket name, object +log_worm_protection() { + if ! get_object_legal_hold "$1" "$2"; then + log 2 "error getting object legal hold status" + return + fi + # shellcheck disable=SC2154 + log 5 "LEGAL HOLD: $legal_hold" + if ! get_object_retention "$1" "$2"; then + log 2 "error getting object retention" + # shellcheck disable=SC2154 + if [[ $get_object_retention_error != *"NoSuchObjectLockConfiguration"* ]]; then + return + fi + fi + # shellcheck disable=SC2154 + log 5 "RETENTION: $retention" +}