From f963fbe734c44f392c7ba308caf34496b8b31a51 Mon Sep 17 00:00:00 2001 From: Luke Date: Tue, 15 Oct 2024 19:35:06 -0300 Subject: [PATCH] test: more versioning, legal hold testing, command reporting (#887) --- .github/workflows/system.yml | 2 + tests/.env.default | 1 + tests/commands/abort_multipart_upload.sh | 4 +- tests/commands/command.sh | 34 +++++++++ tests/commands/complete_multipart_upload.sh | 2 +- tests/commands/copy_object.sh | 10 +-- tests/commands/create_bucket.sh | 16 ++-- tests/commands/create_multipart_upload.sh | 8 +- tests/commands/create_presigned_url.sh | 6 +- tests/commands/delete_bucket.sh | 8 +- tests/commands/delete_bucket_policy.sh | 8 +- tests/commands/delete_bucket_tagging.sh | 6 +- tests/commands/delete_object.sh | 22 +++--- tests/commands/delete_object_tagging.sh | 6 +- tests/commands/delete_objects.sh | 2 +- tests/commands/get_bucket_acl.sh | 6 +- tests/commands/get_bucket_location.sh | 6 +- .../commands/get_bucket_ownership_controls.sh | 2 +- tests/commands/get_bucket_policy.sh | 8 +- tests/commands/get_bucket_tagging.sh | 6 +- tests/commands/get_bucket_versioning.sh | 4 +- tests/commands/get_object.sh | 18 ++--- tests/commands/get_object_attributes.sh | 2 +- tests/commands/get_object_legal_hold.sh | 48 +++++++++++- .../commands/get_object_lock_configuration.sh | 4 +- tests/commands/get_object_retention.sh | 4 +- tests/commands/get_object_tagging.sh | 6 +- tests/commands/head_bucket.sh | 6 +- tests/commands/head_object.sh | 6 +- tests/commands/list_buckets.sh | 16 ++-- tests/commands/list_multipart_uploads.sh | 4 +- tests/commands/list_object_versions.sh | 4 +- tests/commands/list_objects.sh | 23 +++--- tests/commands/list_objects_v2.sh | 2 +- tests/commands/list_parts.sh | 4 +- tests/commands/put_bucket_acl.sh | 11 +-- .../commands/put_bucket_ownership_controls.sh | 12 ++- tests/commands/put_bucket_policy.sh | 8 +- tests/commands/put_bucket_tagging.sh | 6 +- tests/commands/put_bucket_versioning.sh | 4 +- tests/commands/put_object.sh | 12 +-- tests/commands/put_object_legal_hold.sh | 2 +- .../commands/put_object_lock_configuration.sh | 4 +- tests/commands/put_object_retention.sh | 4 +- tests/commands/put_object_tagging.sh | 6 +- tests/commands/put_public_access_block.sh | 2 +- tests/commands/select_object_content.sh | 2 +- tests/commands/upload_part.sh | 2 +- tests/commands/upload_part_copy.sh | 4 +- tests/env.sh | 9 +++ tests/logger.sh | 54 +++++++++++-- tests/rest_scripts/get_object_legal_hold.sh | 75 +++++++++++++++++++ tests/setup.sh | 4 + tests/test_rest.sh | 70 ++++++++++++++++- tests/test_s3api.sh | 30 ++++---- tests/test_user_aws.sh | 10 ++- tests/util.sh | 15 ++-- tests/util_legal_hold.sh | 23 +++++- tests/util_multipart.sh | 31 ++++++++ tests/util_users.sh | 28 +++---- tests/util_versioning.sh | 44 +++++++++++ tests/versity.sh | 5 ++ 62 files changed, 593 insertions(+), 198 deletions(-) create mode 100644 tests/commands/command.sh create mode 100644 tests/rest_scripts/get_object_legal_hold.sh diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 0d35beb3..b98e7550 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -202,6 +202,8 @@ jobs: PASSWORD_TWO: 8901234 TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files REMOVE_TEST_FILE_FOLDER: true + VERSIONING_DIR: ${{ github.workspace }}/versioning + COMMAND_LOG: command.log run: | make testbin export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST diff --git a/tests/.env.default b/tests/.env.default index ce4a71f8..674f7718 100644 --- a/tests/.env.default +++ b/tests/.env.default @@ -28,3 +28,4 @@ PASSWORD_TWO=OPQRSTU TEST_FILE_FOLDER=$PWD/versity-gwtest-files REMOVE_TEST_FILE_FOLDER=true VERSIONING_DIR=/tmp/versioning +COMMAND_LOG=command.log \ No newline at end of file diff --git a/tests/commands/abort_multipart_upload.sh b/tests/commands/abort_multipart_upload.sh index 3fb255f0..38950811 100644 --- a/tests/commands/abort_multipart_upload.sh +++ b/tests/commands/abort_multipart_upload.sh @@ -20,7 +20,7 @@ abort_multipart_upload() { log 2 "'abort multipart upload' command requires bucket, key, upload ID" return 1 fi - if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then + if ! error=$(send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then log 2 "Error aborting upload: $error" return 1 fi @@ -33,7 +33,7 @@ abort_multipart_upload_with_user() { return 1 fi record_command "abort-multipart-upload" "client:s3api" - if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then + if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then log 2 "Error aborting upload: $abort_multipart_upload_error" export abort_multipart_upload_error return 1 diff --git a/tests/commands/command.sh b/tests/commands/command.sh new file mode 100644 index 00000000..2141225b --- /dev/null +++ b/tests/commands/command.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/logger.sh + +send_command() { + if [ $# -eq 0 ]; then + return 1 + fi + if [ -n "$COMMAND_LOG" ]; then + args=(AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" "$@") + if ! mask_arg_array "${args[@]}"; then + return 1 + fi + # shellcheck disable=SC2154 + echo "${masked_args[*]}" >> "$COMMAND_LOG" + "$@" + return $? + fi + "$@" +} \ No newline at end of file diff --git a/tests/commands/complete_multipart_upload.sh b/tests/commands/complete_multipart_upload.sh index c10657a2..62f8ca21 100644 --- a/tests/commands/complete_multipart_upload.sh +++ b/tests/commands/complete_multipart_upload.sh @@ -21,7 +21,7 @@ complete_multipart_upload() { fi log 5 "complete multipart upload id: $3, parts: $4" record_command "complete-multipart-upload" "client:s3api" - error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$? + error=$(send_command aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" --multipart-upload '{"Parts": '"$4"'}' 2>&1) || local completed=$? if [[ $completed -ne 0 ]]; then log 2 "error completing multipart upload: $error" return 1 diff --git a/tests/commands/copy_object.sh b/tests/commands/copy_object.sh index 28297652..75084720 100644 --- a/tests/commands/copy_object.sh +++ b/tests/commands/copy_object.sh @@ -23,14 +23,14 @@ copy_object() { local error record_command "copy-object" "client:$1" if [[ $1 == 's3' ]]; then - error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$? + error=$(send_command aws --no-verify-ssl s3 cp "$2" s3://"$3/$4" 2>&1) || exit_code=$? elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then - error=$(aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$? + error=$(send_command aws --no-verify-ssl s3api copy-object --copy-source "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate cp s3://$2 s3://$3/$4" - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$? + error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate cp "s3://$2" s3://"$3/$4" 2>&1) || exit_code=$? elif [[ $1 == 'mc' ]]; then - error=$(mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$? + error=$(send_command mc --insecure cp "$MC_ALIAS/$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$? else echo "'copy-object' not implemented for '$1'" return 1 @@ -45,7 +45,7 @@ copy_object() { copy_object_empty() { record-command "copy-object" "client:s3api" - error=$(aws --no-verify-ssl s3api copy-object 2>&1) || local result=$? + error=$(send_command aws --no-verify-ssl s3api copy-object 2>&1) || local result=$? if [[ $result -eq 0 ]]; then log 2 "copy object with empty parameters returned no error" return 1 diff --git a/tests/commands/create_bucket.sh b/tests/commands/create_bucket.sh index e046d536..46354710 100644 --- a/tests/commands/create_bucket.sh +++ b/tests/commands/create_bucket.sh @@ -30,14 +30,14 @@ create_bucket() { local error log 6 "create bucket" if [[ $1 == 's3' ]]; then - error=$(aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$? + error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then - error=$(aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$? + error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$2" 2>&1) || exit_code=$? elif [[ $1 == "s3cmd" ]]; then log 5 "s3cmd ${S3CMD_OPTS[*]} --no-check-certificate mb s3://$2" - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$? + error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == "mc" ]]; then - error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? + error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? else log 2 "invalid command type $1" return 1 @@ -56,11 +56,11 @@ create_bucket_with_user() { fi local exit_code=0 if [[ $1 == "aws" ]] || [[ $1 == "s3api" ]]; then - error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$? + error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == "s3cmd" ]]; then - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$? + error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb --access_key="$3" --secret_key="$4" s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == "mc" ]]; then - error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? + error=$(send_command mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? else log 2 "invalid command type $1" return 1 @@ -80,7 +80,7 @@ create_bucket_object_lock_enabled() { fi local exit_code=0 - error=$(aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$? + error=$(send_command aws --no-verify-ssl s3api create-bucket --bucket "$1" 2>&1 --object-lock-enabled-for-bucket) || local exit_code=$? if [ $exit_code -ne 0 ]; then log 2 "error creating bucket: $error" return 1 diff --git a/tests/commands/create_multipart_upload.sh b/tests/commands/create_multipart_upload.sh index a3468db0..46ec4d6a 100644 --- a/tests/commands/create_multipart_upload.sh +++ b/tests/commands/create_multipart_upload.sh @@ -24,7 +24,7 @@ create_multipart_upload() { return 1 fi - if ! multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then + if ! multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then log 2 "Error creating multipart upload: $multipart_data" return 1 fi @@ -44,7 +44,7 @@ create_multipart_upload_with_user() { return 1 fi - if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then + if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then log 2 "Error creating multipart upload: $multipart_data" return 1 fi @@ -65,7 +65,7 @@ create_multipart_upload_params() { return 1 fi local multipart_data - multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload \ + multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload \ --bucket "$1" \ --key "$2" \ --content-type "$3" \ @@ -96,7 +96,7 @@ create_multipart_upload_custom() { done log 5 "${*:3}" log 5 "aws --no-verify-ssl s3api create-multipart-upload --bucket $1 --key $2 ${*:3}" - multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$? + multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1) || local result=$? if [[ $result -ne 0 ]]; then log 2 "error creating custom multipart data command: $multipart_data" return 1 diff --git a/tests/commands/create_presigned_url.sh b/tests/commands/create_presigned_url.sh index f01efb46..c0d0ffd9 100644 --- a/tests/commands/create_presigned_url.sh +++ b/tests/commands/create_presigned_url.sh @@ -8,11 +8,11 @@ create_presigned_url() { local presign_result=0 if [[ $1 == 'aws' ]]; then - presigned_url=$(aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$? + presigned_url=$(send_command aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$? elif [[ $1 == 's3cmd' ]]; then - presigned_url=$(s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$? + presigned_url=$(send_command s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$? elif [[ $1 == 'mc' ]]; then - presigned_url_data=$(mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$? + presigned_url_data=$(send_command mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$? presigned_url="${presigned_url_data#*Share: }" else log 2 "unrecognized command type $1" diff --git a/tests/commands/delete_bucket.sh b/tests/commands/delete_bucket.sh index 9c7ddb9f..60374d30 100644 --- a/tests/commands/delete_bucket.sh +++ b/tests/commands/delete_bucket.sh @@ -31,13 +31,13 @@ delete_bucket() { exit_code=0 if [[ $1 == 's3' ]]; then - error=$(aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$? + error=$(send_command aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$? elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then - error=$(aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$? + error=$(send_command aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$? + error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == 'mc' ]]; then - error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$? + error=$(send_command mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$? else log 2 "Invalid command type $1" return 1 diff --git a/tests/commands/delete_bucket_policy.sh b/tests/commands/delete_bucket_policy.sh index af21243e..643fef3a 100644 --- a/tests/commands/delete_bucket_policy.sh +++ b/tests/commands/delete_bucket_policy.sh @@ -22,11 +22,11 @@ delete_bucket_policy() { fi local delete_result=0 if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then - error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$? + error=$(send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$? elif [[ $1 == 's3cmd' ]]; then - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$? + error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$? elif [[ $1 == 'mc' ]]; then - error=$(mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$? + error=$(send_command mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$? else log 2 "command 'delete bucket policy' not implemented for '$1'" return 1 @@ -44,7 +44,7 @@ delete_bucket_policy_with_user() { log 2 "'delete bucket policy with user' command requires bucket, username, password" return 1 fi - if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then + if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then log 2 "error deleting bucket policy: $delete_bucket_policy_error" export delete_bucket_policy_error return 1 diff --git a/tests/commands/delete_bucket_tagging.sh b/tests/commands/delete_bucket_tagging.sh index 7846de1f..7a28622b 100644 --- a/tests/commands/delete_bucket_tagging.sh +++ b/tests/commands/delete_bucket_tagging.sh @@ -22,9 +22,9 @@ delete_bucket_tagging() { fi local result if [[ $1 == 'aws' ]]; then - tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$? + tags=$(send_command aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$? elif [[ $1 == 'mc' ]]; then - tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$? + tags=$(send_command mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$? else log 2 "invalid command type $1" return 1 @@ -43,7 +43,7 @@ delete_bucket_tagging_with_user() { log 2 "delete bucket tagging command missing username, password, bucket name" return 1 fi - if ! error=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then + if ! error=$(send_command AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$3" 2>&1); then log 2 "error deleting bucket tagging with user: $error" return 1 fi diff --git a/tests/commands/delete_object.sh b/tests/commands/delete_object.sh index e47b27b1..5601ad6f 100644 --- a/tests/commands/delete_object.sh +++ b/tests/commands/delete_object.sh @@ -24,13 +24,13 @@ delete_object() { fi local exit_code=0 if [[ $1 == 's3' ]]; then - delete_object_error=$(aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$? + delete_object_error=$(send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$? elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then - delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$? + delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$? + delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$? elif [[ $1 == 'mc' ]]; then - delete_object_error=$(mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$? + delete_object_error=$(send_command mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$? elif [[ $1 == 'rest' ]]; then delete_object_rest "$2" "$3" || exit_code=$? else @@ -51,7 +51,7 @@ delete_object_bypass_retention() { log 2 "'delete-object with bypass retention' requires bucket, key, user, password" return 1 fi - if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then + if ! delete_object_error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --bypass-governance-retention 2>&1); then log 2 "error deleting object with bypass retention: $delete_object_error" return 1 fi @@ -63,7 +63,7 @@ delete_object_version() { log 2 "'delete_object_version' requires bucket, key, version ID" return 1 fi - if ! delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" 2>&1); then + if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" 2>&1); then log 2 "error deleting object version: $delete_object_error" return 1 fi @@ -75,7 +75,7 @@ delete_object_version_bypass_retention() { log 2 "'delete_object_version_bypass_retention' requires bucket, key, version ID" return 1 fi - if ! delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" --bypass-governance-retention 2>&1); then + if ! delete_object_error=$(send_command aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" --bypass-governance-retention 2>&1); then log 2 "error deleting object version with bypass retention: $delete_object_error" return 1 fi @@ -90,11 +90,11 @@ delete_object_with_user() { fi local exit_code=0 if [[ $1 == 's3' ]]; then - delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$? + delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$? elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then - delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$? + delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$? + delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$? else log 2 "command 'delete object with user' not implemented for '$1'" return 1 @@ -135,7 +135,7 @@ UNSIGNED-PAYLOAD" fi get_signature # shellcheck disable=SC2154 - reply=$(curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2" \ + reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/delete_object_tagging.sh b/tests/commands/delete_object_tagging.sh index d66624e6..6261a00d 100644 --- a/tests/commands/delete_object_tagging.sh +++ b/tests/commands/delete_object_tagging.sh @@ -22,9 +22,9 @@ delete_object_tagging() { fi delete_result=0 if [[ $1 == 'aws' ]]; then - error=$(aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$? + error=$(send_command aws --no-verify-ssl s3api delete-object-tagging --bucket "$2" --key "$3" 2>&1) || delete_result=$? elif [[ $1 == 'mc' ]]; then - error=$(mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$? + error=$(send_command mc --insecure tag remove "$MC_ALIAS/$2/$3") || delete_result=$? elif [ "$1" == 'rest' ]; then delete_object_tagging_rest "$2" "$3" || delete_result=$? else @@ -66,7 +66,7 @@ $payload_hash" fi get_signature # shellcheck disable=SC2154 - reply=$(curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2?tagging" \ + reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2?tagging" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/delete_objects.sh b/tests/commands/delete_objects.sh index b540282f..0f1bbf24 100644 --- a/tests/commands/delete_objects.sh +++ b/tests/commands/delete_objects.sh @@ -20,7 +20,7 @@ delete_objects() { log 2 "'delete-objects' command requires bucket name, two object keys" return 1 fi - if ! error=$(aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{ + if ! error=$(send_command aws --no-verify-ssl s3api delete-objects --bucket "$1" --delete "{ \"Objects\": [ {\"Key\": \"$2\"}, {\"Key\": \"$3\"} diff --git a/tests/commands/get_bucket_acl.sh b/tests/commands/get_bucket_acl.sh index 9705b44b..8f86a6e8 100644 --- a/tests/commands/get_bucket_acl.sh +++ b/tests/commands/get_bucket_acl.sh @@ -22,9 +22,9 @@ get_bucket_acl() { fi local exit_code=0 if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then - acl=$(aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?" + acl=$(send_command aws --no-verify-ssl s3api get-bucket-acl --bucket "$2" 2>&1) || exit_code="$?" elif [[ $1 == 's3cmd' ]]; then - acl=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?" + acl=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$2" 2>&1) || exit_code="$?" else log 2 "command 'get bucket acl' not implemented for $1" return 1 @@ -42,7 +42,7 @@ get_bucket_acl_with_user() { log 2 "'get bucket ACL with user' command requires bucket name, username, password" return 1 fi - if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then + if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then log 2 "error getting bucket ACLs: $bucket_acl" return 1 fi diff --git a/tests/commands/get_bucket_location.sh b/tests/commands/get_bucket_location.sh index cffaf218..09cf3e9e 100644 --- a/tests/commands/get_bucket_location.sh +++ b/tests/commands/get_bucket_location.sh @@ -42,7 +42,7 @@ get_bucket_location_aws() { echo "get bucket location (aws) requires bucket name" return 1 fi - location_json=$(aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$? + location_json=$(send_command aws --no-verify-ssl s3api get-bucket-location --bucket "$1") || location_result=$? if [[ $location_result -ne 0 ]]; then echo "error getting bucket location: $location" return 1 @@ -57,7 +57,7 @@ get_bucket_location_s3cmd() { echo "get bucket location (s3cmd) requires bucket name" return 1 fi - info=$(s3cmd --no-check-certificate info "s3://$1") || results=$? + info=$(send_command s3cmd --no-check-certificate info "s3://$1") || results=$? if [[ $results -ne 0 ]]; then echo "error getting s3cmd info: $info" return 1 @@ -72,7 +72,7 @@ get_bucket_location_mc() { echo "get bucket location (mc) requires bucket name" return 1 fi - info=$(mc --insecure stat "$MC_ALIAS/$1") || results=$? + info=$(send_command mc --insecure stat "$MC_ALIAS/$1") || results=$? if [[ $results -ne 0 ]]; then echo "error getting s3cmd info: $info" return 1 diff --git a/tests/commands/get_bucket_ownership_controls.sh b/tests/commands/get_bucket_ownership_controls.sh index 6fb8d7a7..ff0eb345 100644 --- a/tests/commands/get_bucket_ownership_controls.sh +++ b/tests/commands/get_bucket_ownership_controls.sh @@ -26,7 +26,7 @@ get_bucket_ownership_controls() { return 1 fi - if ! raw_bucket_ownership_controls=$(aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then + if ! raw_bucket_ownership_controls=$(send_command aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls" return 1 fi diff --git a/tests/commands/get_bucket_policy.sh b/tests/commands/get_bucket_policy.sh index 35172be0..822db722 100644 --- a/tests/commands/get_bucket_policy.sh +++ b/tests/commands/get_bucket_policy.sh @@ -44,7 +44,7 @@ get_bucket_policy_aws() { log 2 "aws 'get bucket policy' command requires bucket" return 1 fi - policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$? + policy_json=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$? policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning") log 5 "$policy_json" if [[ $get_result -ne 0 ]]; then @@ -66,7 +66,7 @@ get_bucket_policy_with_user() { log 2 "'get bucket policy with user' command requires bucket, username, password" return 1 fi - if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then + if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning") bucket_policy=$(echo "$policy_json" | jq -r '.Policy') else @@ -87,7 +87,7 @@ get_bucket_policy_s3cmd() { return 1 fi - if ! info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then + if ! info=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1" 2>&1); then log 2 "error getting bucket policy: $info" return 1 fi @@ -129,7 +129,7 @@ get_bucket_policy_mc() { echo "aws 'get bucket policy' command requires bucket" return 1 fi - bucket_policy=$(mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$? + bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$? if [[ $get_result -ne 0 ]]; then echo "error getting policy: $bucket_policy" return 1 diff --git a/tests/commands/get_bucket_tagging.sh b/tests/commands/get_bucket_tagging.sh index af634f4d..fa72c4d2 100644 --- a/tests/commands/get_bucket_tagging.sh +++ b/tests/commands/get_bucket_tagging.sh @@ -22,9 +22,9 @@ get_bucket_tagging() { record_command "get-bucket-tagging" "client:$1" local result if [[ $1 == 'aws' ]]; then - tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$? + tags=$(send_command aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$? elif [[ $1 == 'mc' ]]; then - tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$? + tags=$(send_command mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$? else fail "invalid command type $1" fi @@ -49,7 +49,7 @@ get_bucket_tagging_with_user() { fi record_command "get-bucket-tagging" "client:s3api" local result - if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then + if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then log 5 "tags error: $tags" if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then export tags= diff --git a/tests/commands/get_bucket_versioning.sh b/tests/commands/get_bucket_versioning.sh index 5c81bd9f..ae6ad09d 100644 --- a/tests/commands/get_bucket_versioning.sh +++ b/tests/commands/get_bucket_versioning.sh @@ -22,7 +22,7 @@ get_bucket_versioning() { fi local get_result=0 if [[ $1 == 's3api' ]]; then - versioning=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$? + versioning=$(send_command aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$? fi if [[ $get_result -ne 0 ]]; then log 2 "error getting bucket versioning: $versioning" @@ -60,7 +60,7 @@ UNSIGNED-PAYLOAD" fi get_signature # shellcheck disable=SC2154 - if ! reply=$(curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?versioning" \ + if ! reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?versioning" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/get_object.sh b/tests/commands/get_object.sh index 1cf30589..ae6be41f 100644 --- a/tests/commands/get_object.sh +++ b/tests/commands/get_object.sh @@ -23,13 +23,13 @@ get_object() { fi local exit_code=0 if [[ $1 == 's3' ]]; then - get_object_error=$(aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$? + get_object_error=$(send_command aws --no-verify-ssl s3 mv "s3://$2/$3" "$4" 2>&1) || exit_code=$? elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then - get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$? + get_object_error=$(send_command aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$? + get_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$? elif [[ $1 == 'mc' ]]; then - get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$? + get_object_error=$(send_command mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$? elif [[ $1 == 'rest' ]]; then get_object_rest "$2" "$3" "$4" || exit_code=$? else @@ -50,7 +50,7 @@ get_object_with_range() { log 2 "'get object with range' requires bucket, key, range, outfile" return 1 fi - if ! get_object_error=$(aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1); then + if ! get_object_error=$(send_command aws --no-verify-ssl s3api get-object --bucket "$1" --key "$2" --range "$3" "$4" 2>&1); then log 2 "error getting object with range: $get_object_error" return 1 fi @@ -66,13 +66,13 @@ get_object_with_user() { fi local exit_code=0 if [[ $1 == 's3' ]] || [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then - get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$? + get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" send_command aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$? elif [[ $1 == "s3cmd" ]]; then log 5 "s3cmd filename: $3" - get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$5" --secret_key="$6" get "s3://$2/$3" "$4" 2>&1) || exit_code=$? + get_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$5" --secret_key="$6" get "s3://$2/$3" "$4" 2>&1) || exit_code=$? elif [[ $1 == "mc" ]]; then log 5 "save location: $4" - get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$? + get_object_error=$(send_command mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$? else log 2 "'get_object_with_user' not implemented for client '$1'" return 1 @@ -114,7 +114,7 @@ UNSIGNED-PAYLOAD" fi get_signature # shellcheck disable=SC2154 - reply=$(curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1/$2" \ + reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1/$2" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/get_object_attributes.sh b/tests/commands/get_object_attributes.sh index 0a1cf590..7591b222 100644 --- a/tests/commands/get_object_attributes.sh +++ b/tests/commands/get_object_attributes.sh @@ -20,7 +20,7 @@ get_object_attributes() { log 2 "'get object attributes' command requires bucket, key" return 1 fi - attributes=$(aws --no-verify-ssl s3api get-object-attributes --bucket "$1" --key "$2" --object-attributes "ObjectSize" 2>&1) || local get_result=$? + attributes=$(send_command aws --no-verify-ssl s3api get-object-attributes --bucket "$1" --key "$2" --object-attributes "ObjectSize" 2>&1) || local get_result=$? if [[ $get_result -ne 0 ]]; then log 2 "error getting object attributes: $attributes" return 1 diff --git a/tests/commands/get_object_legal_hold.sh b/tests/commands/get_object_legal_hold.sh index a691c246..b530e586 100644 --- a/tests/commands/get_object_legal_hold.sh +++ b/tests/commands/get_object_legal_hold.sh @@ -20,10 +20,56 @@ get_object_legal_hold() { return 1 fi record_command "get-object-legal-hold" "client:s3api" - legal_hold=$(aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$? + legal_hold=$(send_command aws --no-verify-ssl s3api get-object-legal-hold --bucket "$1" --key "$2" 2>&1) || local get_result=$? if [[ $get_result -ne 0 ]]; then log 2 "error getting object legal hold: $legal_hold" return 1 fi return 0 +} + +get_object_legal_hold_rest() { + if [ $# -ne 2 ]; then + log 2 "'get_object_legal_hold_rest' requires bucket, key" + return 1 + fi + + generate_hash_for_payload "" + + current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} + header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') + # shellcheck disable=SC2154 + canonical_request="GET +/$1/$2 +legal-hold= +host:$aws_endpoint_url_address +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + + if ! generate_sts_string "$current_date_time" "$canonical_request"; then + log 2 "error generating sts string" + return 1 + fi + get_signature + # shellcheck disable=SC2154 + reply=$(send_command curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?legal-hold" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ + -o "$TEST_FILE_FOLDER"/object_legal_hold.txt 2>&1) + log 5 "reply status code: $reply" + if [[ "$reply" != "200" ]]; then + if [ "$reply" == "404" ]; then + return 1 + fi + log 2 "reply error: $reply" + log 2 "get object retention command returned error: $(cat "$TEST_FILE_FOLDER"/object_legal_hold.txt)" + return 2 + fi + log 5 "object legal hold: $(cat "$TEST_FILE_FOLDER"/object_legal_hold.txt)" + return 0 } \ No newline at end of file diff --git a/tests/commands/get_object_lock_configuration.sh b/tests/commands/get_object_lock_configuration.sh index f115dd77..145b13c9 100644 --- a/tests/commands/get_object_lock_configuration.sh +++ b/tests/commands/get_object_lock_configuration.sh @@ -20,7 +20,7 @@ get_object_lock_configuration() { log 2 "'get object lock configuration' command missing bucket name" return 1 fi - if ! lock_config=$(aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then + if ! lock_config=$(send_command aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then log 2 "error obtaining lock config: $lock_config" # shellcheck disable=SC2034 get_object_lock_config_err=$lock_config @@ -57,7 +57,7 @@ UNSIGNED-PAYLOAD" fi get_signature # shellcheck disable=SC2154 - reply=$(curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \ + reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/get_object_retention.sh b/tests/commands/get_object_retention.sh index 83daf584..4a8a6a83 100644 --- a/tests/commands/get_object_retention.sh +++ b/tests/commands/get_object_retention.sh @@ -20,7 +20,7 @@ get_object_retention() { log 2 "'get object retention' command requires bucket, key" return 1 fi - if ! retention=$(aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1); then + if ! retention=$(send_command aws --no-verify-ssl s3api get-object-retention --bucket "$1" --key "$2" 2>&1); then log 2 "error getting object retention: $retention" get_object_retention_error=$retention export get_object_retention_error @@ -57,7 +57,7 @@ $payload_hash" fi get_signature # shellcheck disable=SC2154 - reply=$(curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?retention" \ + reply=$(send_command curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?retention" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/get_object_tagging.sh b/tests/commands/get_object_tagging.sh index d46ccc97..5c6e41ca 100644 --- a/tests/commands/get_object_tagging.sh +++ b/tests/commands/get_object_tagging.sh @@ -22,9 +22,9 @@ get_object_tagging() { fi local result if [[ "$1" == 'aws' ]] || [[ $1 == 's3api' ]]; then - tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$? + tags=$(send_command aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$? elif [[ "$1" == 'mc' ]]; then - tags=$(mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$? + tags=$(send_command mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$? elif [ "$1" == 'rest' ]; then get_object_tagging_rest "$2" "$3" || result=$? else @@ -73,7 +73,7 @@ $payload_hash" fi get_signature # shellcheck disable=SC2154 - reply=$(curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?tagging" \ + reply=$(send_command curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?tagging" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/head_bucket.sh b/tests/commands/head_bucket.sh index 90792661..0531f2ab 100644 --- a/tests/commands/head_bucket.sh +++ b/tests/commands/head_bucket.sh @@ -30,11 +30,11 @@ head_bucket() { fi local exit_code=0 if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then - bucket_info=$(aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$? + bucket_info=$(send_command aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$? elif [[ $1 == "s3cmd" ]]; then - bucket_info=$(s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$? + bucket_info=$(send_command s3cmd --no-check-certificate info "s3://$2" 2>&1) || exit_code=$? elif [[ $1 == 'mc' ]]; then - bucket_info=$(mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? + bucket_info=$(send_command mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? else fail "invalid command type $1" fi diff --git a/tests/commands/head_object.sh b/tests/commands/head_object.sh index 9b2a95d8..16a9bf6b 100644 --- a/tests/commands/head_object.sh +++ b/tests/commands/head_object.sh @@ -22,11 +22,11 @@ head_object() { fi local exit_code=0 if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then - metadata=$(aws --no-verify-ssl s3api head-object --bucket "$2" --key "$3" 2>&1) || exit_code="$?" + metadata=$(send_command aws --no-verify-ssl s3api head-object --bucket "$2" --key "$3" 2>&1) || exit_code="$?" elif [[ $1 == 's3cmd' ]]; then - metadata=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info s3://"$2/$3" 2>&1) || exit_code="$?" + metadata=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info s3://"$2/$3" 2>&1) || exit_code="$?" elif [[ $1 == 'mc' ]]; then - metadata=$(mc --insecure stat "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$? + metadata=$(send_command mc --insecure stat "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$? else log 2 "invalid command type $1" return 2 diff --git a/tests/commands/list_buckets.sh b/tests/commands/list_buckets.sh index 6e3a171a..3afa77ad 100644 --- a/tests/commands/list_buckets.sh +++ b/tests/commands/list_buckets.sh @@ -24,13 +24,13 @@ list_buckets() { local exit_code=0 if [[ $1 == 's3' ]]; then - buckets=$(aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$? + buckets=$(send_command aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$? elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then list_buckets_s3api "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - buckets=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$? + buckets=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$? elif [[ $1 == 'mc' ]]; then - buckets=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$? + buckets=$(send_command mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$? elif [[ $1 == 'rest' ]]; then list_buckets_rest || exit_code=$? else @@ -63,13 +63,13 @@ list_buckets_with_user() { local exit_code=0 if [[ $1 == 's3' ]]; then - buckets=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$? + buckets=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3 ls 2>&1 s3://) || exit_code=$? elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then list_buckets_s3api "$2" "$3" || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - buckets=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$2" --secret_key="$3" ls s3:// 2>&1) || exit_code=$? + buckets=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --access_key="$2" --secret_key="$3" ls s3:// 2>&1) || exit_code=$? elif [[ $1 == 'mc' ]]; then - buckets=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$? + buckets=$(send_command mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$? else echo "list buckets command not implemented for '$1'" return 1 @@ -96,7 +96,7 @@ list_buckets_s3api() { log 2 "'list_buckets_s3api' requires username, password" return 1 fi - if ! output=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api list-buckets 2>&1); then + if ! output=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api list-buckets 2>&1); then echo "error listing buckets: $output" return 1 fi @@ -138,7 +138,7 @@ $payload_hash" get_signature # shellcheck disable=SC2034,SC2154 - reply=$(curl -ks "$AWS_ENDPOINT_URL" \ + reply=$(send_command curl -ks "$AWS_ENDPOINT_URL" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ -H "x-amz-date: $current_date_time" 2>&1) diff --git a/tests/commands/list_multipart_uploads.sh b/tests/commands/list_multipart_uploads.sh index 434ead2d..9720a005 100644 --- a/tests/commands/list_multipart_uploads.sh +++ b/tests/commands/list_multipart_uploads.sh @@ -20,7 +20,7 @@ list_multipart_uploads() { log 2 "'list multipart uploads' command requires bucket name" return 1 fi - if ! uploads=$(aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then + if ! uploads=$(send_command aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then log 2 "error listing uploads: $uploads" return 1 fi @@ -32,7 +32,7 @@ list_multipart_uploads_with_user() { log 2 "'list multipart uploads' command requires bucket name, username, password" return 1 fi - if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then + if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" send_command aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then log 2 "error listing uploads: $uploads" # shellcheck disable=SC2034 list_multipart_uploads_error=$uploads diff --git a/tests/commands/list_object_versions.sh b/tests/commands/list_object_versions.sh index 5da71350..f704b1b6 100644 --- a/tests/commands/list_object_versions.sh +++ b/tests/commands/list_object_versions.sh @@ -20,7 +20,7 @@ list_object_versions() { log 2 "'list object versions' command requires bucket name" return 1 fi - versions=$(aws --no-verify-ssl s3api list-object-versions --bucket "$1" 2>&1) || local list_result=$? + versions=$(send_command aws --no-verify-ssl s3api list-object-versions --bucket "$1" 2>&1) || local list_result=$? if [[ $list_result -ne 0 ]]; then log 2 "error listing object versions: $versions" return 1 @@ -55,7 +55,7 @@ $payload_hash" get_signature # shellcheck disable=SC2034,SC2154 - reply=$(curl -ks "$AWS_ENDPOINT_URL/$1?versions" \ + reply=$(send_command curl -ks "$AWS_ENDPOINT_URL/$1?versions" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/list_objects.sh b/tests/commands/list_objects.sh index 018ee6c8..fe40410d 100644 --- a/tests/commands/list_objects.sh +++ b/tests/commands/list_objects.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash source ./tests/util_list_objects.sh +source ./tests/commands/command.sh # Copyright 2024 Versity Software # This file is licensed under the Apache License, Version 2.0 @@ -29,14 +30,14 @@ list_objects() { local output local result=0 if [[ $1 == "aws" ]] || [[ $1 == 's3' ]]; then - output=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || result=$? + output=$(send_command aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || result=$? elif [[ $1 == 's3api' ]]; then list_objects_s3api "$2" || result=$? return $result elif [[ $1 == 's3cmd' ]]; then - output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || result=$? + output=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || result=$? elif [[ $1 == 'mc' ]]; then - output=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || result=$? + output=$(send_command mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || result=$? elif [[ $1 == 'rest' ]]; then list_objects_rest "$2" || result=$? return $result @@ -66,7 +67,7 @@ list_objects_s3api() { log 2 "'list_objects_s3api' requires bucket" return 1 fi - if ! output=$(aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1); then + if ! output=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1); then log 2 "error listing objects: $output" return 1 fi @@ -94,9 +95,9 @@ list_objects_s3api_v1() { return 1 fi if [ "$2" == "" ]; then - objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$? + objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$? else - objects=$(aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$? + objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$? fi if [[ $result -ne 0 ]]; then echo "error listing objects: $objects" @@ -112,13 +113,13 @@ list_objects_with_prefix() { fi local result=0 if [ "$1" == 's3' ]; then - objects=$(aws --no-verify-ssl s3 ls s3://"$2/$3" 2>&1) || result=$? + objects=$(send_command aws --no-verify-ssl s3 ls s3://"$2/$3" 2>&1) || result=$? elif [ "$1" == 's3api' ]; then - objects=$(aws --no-verify-ssl s3api list-objects --bucket "$2" --prefix "$3" 2>&1) || result=$? + objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$2" --prefix "$3" 2>&1) || result=$? elif [ "$1" == 's3cmd' ]; then - objects=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2/$3" 2>&1) || result=$? + objects=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2/$3" 2>&1) || result=$? elif [[ "$1" == 'mc' ]]; then - objects=$(mc --insecure ls "$MC_ALIAS/$2/$3" 2>&1) || result=$? + objects=$(send_command mc --insecure ls "$MC_ALIAS/$2/$3" 2>&1) || result=$? else log 2 "invalid command type '$1'" return 1 @@ -162,7 +163,7 @@ $payload_hash" fi get_signature # shellcheck disable=SC2154 - reply=$(curl -ks "$header://$aws_endpoint_url_address/$1" \ + reply=$(send_command curl -ks "$header://$aws_endpoint_url_address/$1" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ -H "x-amz-date: $current_date_time" 2>&1) diff --git a/tests/commands/list_objects_v2.sh b/tests/commands/list_objects_v2.sh index ecfd4689..2d9c47a8 100644 --- a/tests/commands/list_objects_v2.sh +++ b/tests/commands/list_objects_v2.sh @@ -23,7 +23,7 @@ list_objects_v2() { return 1 fi record_command "list-objects-v2 client:s3api" - objects=$(aws --no-verify-ssl s3api list-objects-v2 --bucket "$1") || local result=$? + objects=$(send_command aws --no-verify-ssl s3api list-objects-v2 --bucket "$1") || local result=$? if [[ $result -ne 0 ]]; then echo "error listing objects: $objects" return 1 diff --git a/tests/commands/list_parts.sh b/tests/commands/list_parts.sh index 752ac96e..a8ba07ad 100644 --- a/tests/commands/list_parts.sh +++ b/tests/commands/list_parts.sh @@ -20,7 +20,7 @@ list_parts() { return 1 fi record_command "list-parts" "client:s3api" - if ! listed_parts=$(aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then + if ! listed_parts=$(send_command aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then log 2 "Error listing multipart upload parts: $listed_parts" return 1 fi @@ -32,7 +32,7 @@ list_parts_with_user() { return 1 fi record_command 'list-parts' 'client:s3api' - if ! listed_parts=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api list-parts --bucket "$3" --key "$4" --upload-id "$5" 2>&1); then + if ! listed_parts=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" send_command aws --no-verify-ssl s3api list-parts --bucket "$3" --key "$4" --upload-id "$5" 2>&1); then log 2 "Error listing multipart upload parts: $listed_parts" return 1 fi diff --git a/tests/commands/put_bucket_acl.sh b/tests/commands/put_bucket_acl.sh index 457e3053..ba303c7a 100644 --- a/tests/commands/put_bucket_acl.sh +++ b/tests/commands/put_bucket_acl.sh @@ -15,6 +15,7 @@ # under the License. source ./tests/util_file.sh +source ./tests/commands/command.sh put_bucket_acl_s3api() { log 6 "put_bucket_acl_s3api" @@ -24,7 +25,7 @@ put_bucket_acl_s3api() { return 1 fi log 5 "bucket name: $1, acls: $2" - if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then + if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then log 2 "error putting bucket acl: $error" return 1 fi @@ -39,7 +40,7 @@ put_bucket_acl_s3api_with_user() { return 1 fi log 5 "bucket name: $1, acls: $2" - if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then + if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then log 2 "error putting bucket acl: $error" return 1 fi @@ -87,7 +88,7 @@ put_bucket_canned_acl_s3cmd() { log 2 "put bucket acl command requires bucket name, permission" return 1 fi - if ! error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$1" "$2" 2>&1); then + if ! error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$1" "$2" 2>&1); then log 2 "error putting s3cmd canned ACL: $error" return 1 fi @@ -100,7 +101,7 @@ put_bucket_canned_acl() { return 1 fi record_command "put-bucket-acl" "client:s3api" - if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then + if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then log 2 "error re-setting bucket acls: $error" return 1 fi @@ -113,7 +114,7 @@ put_bucket_canned_acl_with_user() { return 1 fi record_command "put-bucket-acl" "client:s3api" - if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then + if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2" 2>&1); then log 2 "error re-setting bucket acls: $error" return 1 fi diff --git a/tests/commands/put_bucket_ownership_controls.sh b/tests/commands/put_bucket_ownership_controls.sh index a5434cbe..5e8ecbdc 100644 --- a/tests/commands/put_bucket_ownership_controls.sh +++ b/tests/commands/put_bucket_ownership_controls.sh @@ -22,9 +22,13 @@ put_bucket_ownership_controls() { fi log 6 "put_bucket_ownership_controls" + if [ $# -ne 2 ]; then + log 2 "'put_bucket_ownership_controls' requires bucket name, rule" + return 1 + fi record_command "put-bucket-ownership-controls" "client:s3api" - assert [ $# -eq 2 ] - run aws --no-verify-ssl s3api put-bucket-ownership-controls --bucket "$1" --ownership-controls="Rules=[{ObjectOwnership=$2}]" - # shellcheck disable=SC2154 - assert_success "error putting bucket ownership controls: $output" + if ! error=$(send_command aws --no-verify-ssl s3api put-bucket-ownership-controls --bucket "$1" --ownership-controls="Rules=[{ObjectOwnership=$2}]" 2>&1); then + log 2 "error putting bucket ownership controls: $error" + return 1 + fi } \ No newline at end of file diff --git a/tests/commands/put_bucket_policy.sh b/tests/commands/put_bucket_policy.sh index f6b72142..457b1612 100644 --- a/tests/commands/put_bucket_policy.sh +++ b/tests/commands/put_bucket_policy.sh @@ -22,11 +22,11 @@ put_bucket_policy() { fi local put_policy_result=0 if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then - policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_policy_result=$? + policy=$(send_command aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_policy_result=$? elif [[ $1 == 's3cmd' ]]; then - policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$? + policy=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$? elif [[ $1 == 'mc' ]]; then - policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_policy_result=$? + policy=$(send_command mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_policy_result=$? else log 2 "command 'put bucket policy' not implemented for '$1'" return 1 @@ -46,7 +46,7 @@ put_bucket_policy_with_user() { log 2 "'put bucket policy with user' command requires bucket, policy file, username, password" return 1 fi - if ! policy=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-policy --bucket "$1" --policy "file://$2" 2>&1); then + if ! policy=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api put-bucket-policy --bucket "$1" --policy "file://$2" 2>&1); then log 2 "error putting bucket policy with user $3: $policy" put_bucket_policy_error=$policy export put_bucket_policy_error diff --git a/tests/commands/put_bucket_tagging.sh b/tests/commands/put_bucket_tagging.sh index 9f89e057..a5e31cf6 100644 --- a/tests/commands/put_bucket_tagging.sh +++ b/tests/commands/put_bucket_tagging.sh @@ -24,9 +24,9 @@ put_bucket_tagging() { local result=0 record_command "put-bucket-tagging" "client:$1" if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then - error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$? + error=$(send_command aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$? elif [[ $1 == 'mc' ]]; then - error=$(mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$? + error=$(send_command mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$? else log 2 "invalid command type $1" return 1 @@ -42,7 +42,7 @@ put_bucket_tagging_with_user() { log 6 "put_bucket_tagging_with_user" assert [ $# -eq 5 ] record_command "put-bucket-tagging" "client:$1" - if ! error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]"); then + if ! error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]"); then log 2 "error putting bucket tagging: $error" return 1 fi diff --git a/tests/commands/put_bucket_versioning.sh b/tests/commands/put_bucket_versioning.sh index c5ba774c..4b390047 100644 --- a/tests/commands/put_bucket_versioning.sh +++ b/tests/commands/put_bucket_versioning.sh @@ -22,7 +22,7 @@ put_bucket_versioning() { fi local put_result=0 if [[ $1 == 's3api' ]]; then - error=$(aws --no-verify-ssl s3api put-bucket-versioning --bucket "$2" --versioning-configuration "{ \"Status\": \"$3\"}" 2>&1) || put_result=$? + error=$(send_command aws --no-verify-ssl s3api put-bucket-versioning --bucket "$2" --versioning-configuration "{ \"Status\": \"$3\"}" 2>&1) || put_result=$? fi if [[ $put_result -ne 0 ]]; then log 2 "error putting bucket versioning: $error" @@ -67,7 +67,7 @@ $payload_hash" get_signature # shellcheck disable=SC2154 - reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1?versioning" \ + reply=$(send_command curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1?versioning" \ -H "Content-MD5: $content_md5" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ diff --git a/tests/commands/put_object.sh b/tests/commands/put_object.sh index 450b5aee..35ba3f00 100644 --- a/tests/commands/put_object.sh +++ b/tests/commands/put_object.sh @@ -26,13 +26,13 @@ put_object() { local exit_code=0 local error if [[ $1 == 's3' ]]; then - error=$(aws --no-verify-ssl s3 mv "$2" s3://"$3/$4" 2>&1) || exit_code=$? + error=$(send_command aws --no-verify-ssl s3 mv "$2" s3://"$3/$4" 2>&1) || exit_code=$? elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then - error=$(aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$? + error=$(send_command aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$3/$4" 2>&1) || exit_code=$? + error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$3/$4" 2>&1) || exit_code=$? elif [[ $1 == 'mc' ]]; then - error=$(mc --insecure put "$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$? + error=$(send_command mc --insecure put "$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$? elif [[ $1 == 'rest' ]]; then put_object_rest "$2" "$3" "$4" || exit_code=$? else @@ -55,7 +55,7 @@ put_object_with_user() { fi local exit_code=0 if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then - put_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$? + put_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" send_command aws --no-verify-ssl s3api put-object --body "$2" --bucket "$3" --key "$4" 2>&1) || exit_code=$? else log 2 "'put object with user' command not implemented for '$1'" return 1 @@ -97,7 +97,7 @@ $payload_hash" fi get_signature # shellcheck disable=SC2154 - reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$2/$3" \ + reply=$(send_command curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$2/$3" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/put_object_legal_hold.sh b/tests/commands/put_object_legal_hold.sh index e1f25a8f..8a65fd67 100644 --- a/tests/commands/put_object_legal_hold.sh +++ b/tests/commands/put_object_legal_hold.sh @@ -21,7 +21,7 @@ put_object_legal_hold() { return 1 fi local error="" - error=$(aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --legal-hold "{\"Status\": \"$3\"}" 2>&1) || local put_hold_result=$? + error=$(send_command aws --no-verify-ssl s3api put-object-legal-hold --bucket "$1" --key "$2" --legal-hold "{\"Status\": \"$3\"}" 2>&1) || local put_hold_result=$? if [[ $put_hold_result -ne 0 ]]; then log 2 "error putting object legal hold: $error" return 1 diff --git a/tests/commands/put_object_lock_configuration.sh b/tests/commands/put_object_lock_configuration.sh index 6f92d76e..c0ad0073 100644 --- a/tests/commands/put_object_lock_configuration.sh +++ b/tests/commands/put_object_lock_configuration.sh @@ -20,7 +20,7 @@ put_object_lock_configuration() { return 1 fi local config="{\"ObjectLockEnabled\": \"$2\", \"Rule\": {\"DefaultRetention\": {\"Mode\": \"$3\", \"Days\": $4}}}" - if ! error=$(aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then + if ! error=$(send_command aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then log 2 "error putting object lock configuration: $error" return 1 fi @@ -33,7 +33,7 @@ put_object_lock_configuration_disabled() { return 1 fi local config="{\"ObjectLockEnabled\": \"Enabled\"}" - if ! error=$(aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then + if ! error=$(send_command aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then log 2 "error putting object lock configuration: $error" return 1 fi diff --git a/tests/commands/put_object_retention.sh b/tests/commands/put_object_retention.sh index aa6ed76d..e329fb2a 100644 --- a/tests/commands/put_object_retention.sh +++ b/tests/commands/put_object_retention.sh @@ -20,7 +20,7 @@ put_object_retention() { log 2 "'put object retention' command requires bucket, key, retention mode, retention date" return 1 fi - error=$(aws --no-verify-ssl s3api put-object-retention --bucket "$1" --key "$2" --retention "{\"Mode\": \"$3\", \"RetainUntilDate\": \"$4\"}" 2>&1) || local put_result=$? + error=$(send_command aws --no-verify-ssl s3api put-object-retention --bucket "$1" --key "$2" --retention "{\"Mode\": \"$3\", \"RetainUntilDate\": \"$4\"}" 2>&1) || local put_result=$? if [[ $put_result -ne 0 ]]; then log 2 "error putting object retention: $error" return 1 @@ -66,7 +66,7 @@ $payload_hash" get_signature # shellcheck disable=SC2154 - reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?retention" \ + reply=$(send_command curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?retention" \ -H "Content-MD5: $content_md5" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ diff --git a/tests/commands/put_object_tagging.sh b/tests/commands/put_object_tagging.sh index 860bb0fd..80217d84 100644 --- a/tests/commands/put_object_tagging.sh +++ b/tests/commands/put_object_tagging.sh @@ -23,9 +23,9 @@ put_object_tagging() { local result record_command "put-object-tagging" "client:$1" if [[ $1 == 'aws' ]]; then - error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$? + error=$(send_command aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$? elif [[ $1 == 'mc' ]]; then - error=$(mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$? + error=$(send_command mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$? elif [[ $1 == 'rest' ]]; then put_object_tagging_rest "$2" "$3" "$4" "$5" || result=$? else @@ -77,7 +77,7 @@ $payload_hash" fi get_signature # shellcheck disable=SC2154 - reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?tagging" \ + reply=$(send_command curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?tagging" \ -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ -H "x-amz-content-sha256: $payload_hash" \ -H "x-amz-date: $current_date_time" \ diff --git a/tests/commands/put_public_access_block.sh b/tests/commands/put_public_access_block.sh index 6f4777d4..ffcd8c17 100644 --- a/tests/commands/put_public_access_block.sh +++ b/tests/commands/put_public_access_block.sh @@ -19,7 +19,7 @@ put_public_access_block() { log 2 "'put_public_access_block' command requires bucket, access block list" return 1 fi - if ! error=$(aws --no-verify-ssl s3api put-public-access-block --bucket "$1" --public-access-block-configuration "$2"); then + if ! error=$(send_command aws --no-verify-ssl s3api put-public-access-block --bucket "$1" --public-access-block-configuration "$2"); then log 2 "error updating public access block: $error" return 1 fi diff --git a/tests/commands/select_object_content.sh b/tests/commands/select_object_content.sh index 05671b15..a10e0368 100644 --- a/tests/commands/select_object_content.sh +++ b/tests/commands/select_object_content.sh @@ -20,7 +20,7 @@ select_object_content() { log 2 "'select object content' command requires bucket, key, expression, expression type, input serialization, output serialization, outfile" return 1 fi - error=$(aws --no-verify-ssl s3api select-object-content \ + error=$(send_command aws --no-verify-ssl s3api select-object-content \ --bucket "$1" \ --key "$2" \ --expression "$3" \ diff --git a/tests/commands/upload_part.sh b/tests/commands/upload_part.sh index e2cededd..54e8bdf1 100644 --- a/tests/commands/upload_part.sh +++ b/tests/commands/upload_part.sh @@ -21,7 +21,7 @@ upload_part() { fi local etag_json record_command "upload-part" "client:s3api" - if ! etag_json=$(aws --no-verify-ssl s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))" 2>&1); then + if ! etag_json=$(send_command aws --no-verify-ssl s3api upload-part --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --body "$4-$(($5-1))" 2>&1); then log 2 "Error uploading part $5: $etag_json" return 1 fi diff --git a/tests/commands/upload_part_copy.sh b/tests/commands/upload_part_copy.sh index 9f572919..50d35983 100644 --- a/tests/commands/upload_part_copy.sh +++ b/tests/commands/upload_part_copy.sh @@ -22,7 +22,7 @@ upload_part_copy() { fi local etag_json echo "$1 $2 $3 $4 $5" - etag_json=$(aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$? + etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$? if [[ $uploaded -ne 0 ]]; then echo "Error uploading part $5: $etag_json" return 1 @@ -39,7 +39,7 @@ upload_part_copy_with_range() { fi local etag_json log 5 "bucket: $1, key: $2, upload ID: $3, file name: $4, range: $5, copy source range: $6" - etag_json=$(aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" --copy-source-range "$6" 2>&1) || local uploaded=$? + etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" --copy-source-range "$6" 2>&1) || local uploaded=$? if [[ $uploaded -ne 0 ]]; then log 2 "Error uploading part $5: $etag_json" export upload_part_copy_error=$etag_json diff --git a/tests/env.sh b/tests/env.sh index f9a36345..6f78eb04 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -62,6 +62,15 @@ check_universal_vars() { if [[ $BYPASS_ENV_FILE != "true" ]]; then source_config_file fi + if [ -n "$COMMAND_LOG" ]; then + if [ -e "$COMMAND_LOG" ]; then + if ! error=$(rm "$COMMAND_LOG"); then + log 3 "error removing command log: $error" + return 1 + fi + fi + echo "******** $(date +"%Y-%m-%d %H:%M:%S") $BATS_TEST_NAME COMMANDS ********" >> "$COMMAND_LOG" + fi if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then # shellcheck source=./tests/.secrets diff --git a/tests/logger.sh b/tests/logger.sh index 39994789..72f29321 100644 --- a/tests/logger.sh +++ b/tests/logger.sh @@ -37,7 +37,7 @@ log() { 6) log_level="TRACE";; *) echo "invalid log level $1"; return 1 esac - if [[ "$2" == *"secret"* ]]; then + if [[ ( "$2" == *"access"* ) || ( "$2" == *"secret"* ) || ( "$2" == *"Credential="* ) ]]; then log_mask "$log_level" "$2" return 0 fi @@ -49,18 +49,42 @@ log_mask() { echo "mask and log requires level, string" return 1 fi + + if ! mask_args "$2"; then + echo "error masking args" + return 1 + fi + + log_message "$log_level" "${masked_args[*]}" +} + +mask_args() { + if [ $# -ne 1 ]; then + echo "'mask_args' requires string" + return 1 + fi + IFS=' ' read -r -a array <<< "$1" + + if ! mask_arg_array "${array[@]}"; then + echo "error masking arg array" + return 1 + fi +} + +mask_arg_array() { masked_args=() # Initialize an array to hold the masked arguments - - IFS=' ' read -r -a array <<< "$2" - + if [ $# -eq 0 ]; then + echo "'mask_arg_array' requires parameters" + return 1 + fi mask_next=false - for arg in "${array[@]}"; do + is_access=false + for arg in "$@"; do if ! check_arg_for_mask "$arg"; then echo "error checking arg for mask" return 1 fi done - log_message "$log_level" "${masked_args[*]}" } check_arg_for_mask() { @@ -69,15 +93,31 @@ check_arg_for_mask() { return 1 fi if [[ $mask_next == true ]]; then - masked_args+=("********") + if [ "$is_access" == "true" ]; then + masked_args+=("${arg:0:4}****") + is_access=false + else + masked_args+=("********") + fi mask_next=false elif [[ "$arg" == --secret_key=* ]]; then masked_args+=("--secret_key=********") elif [[ "$arg" == --secret=* ]]; then masked_args+=("--secret=********") + elif [[ "$arg" == --access=* ]]; then + masked_args+=("${arg:0:13}****") + elif [[ "$arg" == --access_key=* ]]; then + masked_args+=("${arg:0:17}****") + elif [[ "$arg" == *"Credential="* ]]; then + masked_args+=("$(echo "$arg" | sed -E 's/(Credential=[A-Z]{4})[^\/]*/\1****/g')") + elif [[ "$arg" == *"AWS_ACCESS_KEY_ID="* ]]; then + masked_args+=("AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:0:4}****") else if [[ "$arg" == "--secret_key" ]] || [[ "$arg" == "--secret" ]] || [[ "$arg" == "--s3-iam-secret" ]]; then mask_next=true + elif [[ ( "$arg" == "--access" ) || ( "$arg" == "--owner") ]]; then + mask_next=true + is_access=true fi masked_args+=("$arg") fi diff --git a/tests/rest_scripts/get_object_legal_hold.sh b/tests/rest_scripts/get_object_legal_hold.sh new file mode 100644 index 00000000..a7c97727 --- /dev/null +++ b/tests/rest_scripts/get_object_legal_hold.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +payload="" +# shellcheck disable=SC2153 +aws_access_key_id="$AWS_ACCESS_KEY_ID" +# shellcheck disable=SC2153 +aws_secret_access_key="$AWS_SECRET_ACCESS_KEY" +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +get_host +get_aws_region + +# Step 1: generate payload hash + +payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" + +# Step 2: generate canonical hash + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="GET +/$bucket_name +legal-hold= +host:$host +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + +canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')" + +# Step 3: create STS data string + +year_month_day="$(echo "$current_date_time" | cut -c1-8)" + +sts_data="AWS4-HMAC-SHA256 +$current_date_time +$year_month_day/$aws_region/s3/aws4_request +$canonical_request_hash" + +# Step 4: generate signature + +date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}') +date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}') +date_region_service_key=$(echo -n "s3" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_key" | awk '{print $2}') +signing_key=$(echo -n "aws4_request" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_region_service_key" | awk '{print $2}') +signature=$(echo -n "$sts_data" | openssl dgst -sha256 \ + -mac HMAC \ + -macopt hexkey:"$signing_key" | awk '{print $2}') + +# Step 5: send curl command + +curl -ks "https://$host/$BUCKET_NAME?legal-hold" \ + -H "Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ + -H "x-amz-content-sha256: $payload_hash" \ + -H "x-amz-date: $current_date_time" \ No newline at end of file diff --git a/tests/setup.sh b/tests/setup.sh index c28a9ccb..8eb1ecea 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -57,6 +57,10 @@ setup() { # bats teardown function teardown() { + if [[ ( "$BATS_TEST_COMPLETED" -ne 1 ) && ( -e "$COMMAND_LOG" ) ]]; then + cat "$COMMAND_LOG" + echo "**********************************************************************************" + fi # shellcheck disable=SC2154 if ! delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME"; then log 3 "error deleting bucket $BUCKET_ONE_NAME or contents" diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 21bfc6b2..6f115def 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -14,6 +14,7 @@ source ./tests/commands/put_object_tagging.sh source ./tests/logger.sh source ./tests/setup.sh source ./tests/util.sh +source ./tests/util_legal_hold.sh source ./tests/util_list_buckets.sh source ./tests/util_list_objects.sh source ./tests/util_lock_config.sh @@ -124,9 +125,7 @@ source ./tests/util_versioning.sh assert_success } -@test "test_rest_set_get_versioning" { - skip "https://github.com/versity/versitygw/issues/866" - +@test "REST - check, enable, suspend versioning" { run setup_bucket "s3api" "$BUCKET_ONE_NAME" assert_success @@ -200,3 +199,68 @@ source ./tests/util_versioning.sh run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "2" "true" "false" "false" "true" assert_success } + +@test "versioning - add version, then delete and check for marker" { + skip "https://github.com/versity/versitygw/issues/864" + test_file="test_file" + + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run create_test_file "$test_file" + assert_success + + run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled" + assert_success + + run delete_object_rest "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run check_versions_after_file_deletion "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "versioning - retrieve after delete" { + skip "https://github.com/versity/versitygw/issues/888" + test_file="test_file" + + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run create_test_file "$test_file" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled" + assert_success + + run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + assert_failure +} + +@test "REST - legal hold, get without config" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/883" + fi + test_file="test_file" + + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run create_test_file "$test_file" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run check_legal_hold_without_lock_enabled "$BUCKET_ONE_NAME" "$test_file" + assert_success +} diff --git a/tests/test_s3api.sh b/tests/test_s3api.sh index 7b856d71..040a3a62 100755 --- a/tests/test_s3api.sh +++ b/tests/test_s3api.sh @@ -108,6 +108,9 @@ export RUN_USERS=true # delete-objects @test "test_delete_objects" { + if [ "$RECREATE_BUCKETS" == "false" ]; then + skip "https://github.com/versity/versitygw/issues/888" + fi test_delete_objects_aws_root } @@ -151,10 +154,16 @@ export RUN_USERS=true # test adding and removing an object on versitygw @test "test_put_object_with_data" { + if [ "$RECREATE_BUCKETS" == "false" ]; then + skip "https://github.com/versity/versitygw/issues/888" + fi test_common_put_object_with_data "aws" } @test "test_put_object_no_data" { + if [ "$RECREATE_BUCKETS" == "false" ]; then + skip "https://github.com/versity/versitygw/issues/888" + fi test_common_put_object_no_data "aws" } @@ -216,19 +225,8 @@ export RUN_USERS=true run setup_bucket "aws" "$BUCKET_ONE_NAME" assert_success - create_and_list_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two" - - local key_one - local key_two - # shellcheck disable=SC2154 - log 5 "Uploads: $uploads" - raw_uploads=$(echo "$uploads" | grep -v "InsecureRequestWarning") - key_one=$(echo "$raw_uploads" | jq -r '.Uploads[0].Key' 2>&1) || fail "error getting key one: $key_one" - key_two=$(echo "$raw_uploads" | jq -r '.Uploads[1].Key' 2>&1) || fail "error getting key two: $key_two" - key_one=${key_one//\"/} - key_two=${key_two//\"/} - [[ "$bucket_file_one" == *"$key_one" ]] || fail "Key mismatch ($bucket_file_one, $key_one)" - [[ "$bucket_file_two" == *"$key_two" ]] || fail "Key mismatch ($bucket_file_two, $key_two)" + run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two" + assert_success } @test "test-multipart-upload-from-bucket" { @@ -398,8 +396,10 @@ export RUN_USERS=true @test "test_put_object_lock_configuration" { bucket_name=$BUCKET_ONE_NAME if [[ $RECREATE_BUCKETS == "true" ]]; then - delete_bucket "s3api" "$bucket_name" || fail "error deleting bucket" - create_bucket_object_lock_enabled "$bucket_name" || fail "error setting up bucket" + run delete_bucket "s3api" "$bucket_name" + assert_success + run create_bucket_object_lock_enabled "$bucket_name" + assert_success fi local enabled="Enabled" local governance="GOVERNANCE" diff --git a/tests/test_user_aws.sh b/tests/test_user_aws.sh index 7d87e108..bfa28c3a 100755 --- a/tests/test_user_aws.sh +++ b/tests/test_user_aws.sh @@ -107,6 +107,9 @@ export RUN_USERS=true } @test "test_admin_put_get_object" { + if [ "$RECREATE_BUCKETS" == "false" ]; then + skip "https://github.com/versity/versitygw/issues/888" + fi username="$USERNAME_ONE" password="$PASSWORD_ONE" test_file="test_file" @@ -122,7 +125,12 @@ export RUN_USERS=true put_object_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password" || fail "failed to add object to bucket" get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password" || fail "error getting object" compare_files "$TEST_FILE_FOLDER/$test_file" "$TEST_FILE_FOLDER/$test_file-copy" || fail "files don't match" - delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password" || fail "error deleting object" + list_object_versions "$BUCKET_ONE_NAME" + log 5 "versions: $versions" + run delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password" + assert_success + list_object_versions "$BUCKET_ONE_NAME" + log 5 "versions: $versions" if get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"; then fail "file not successfully deleted" fi diff --git a/tests/util.sh b/tests/util.sh index 1293dff6..8cc739d3 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -41,6 +41,7 @@ source ./tests/commands/list_parts.sh source ./tests/commands/put_bucket_acl.sh source ./tests/commands/put_bucket_ownership_controls.sh source ./tests/commands/put_bucket_policy.sh +source ./tests/commands/put_bucket_versioning.sh source ./tests/commands/put_object_legal_hold.sh source ./tests/commands/put_object_lock_configuration.sh source ./tests/commands/upload_part_copy.sh @@ -102,7 +103,7 @@ add_governance_bypass_policy() { fi cat < "$TEST_FILE_FOLDER/policy-bypass-governance.txt" { - "Version": "dummy", + "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", @@ -223,13 +224,13 @@ clear_bucket_s3api() { return 1 fi - if ! list_and_delete_objects "$1"; then - log 2 "error listing and deleting objects" + if [[ "$DIRECT" != "true" ]] && ! add_governance_bypass_policy "$1"; then + log 2 "error adding governance bypass policy" return 1 fi - if ! delete_bucket_policy "s3api" "$1"; then - log 2 "error deleting bucket policy" + if ! list_and_delete_objects "$1"; then + log 2 "error listing and deleting objects" return 1 fi @@ -240,6 +241,10 @@ clear_bucket_s3api() { log 2 "error disabling object lock config" return 1 fi + #if ! put_bucket_versioning "s3api" "$1" "Suspended"; then + # log 2 "error suspending bucket versioning" + # return 1 + #fi #if ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID"; then # log 2 "error changing bucket owner back to root" diff --git a/tests/util_legal_hold.sh b/tests/util_legal_hold.sh index 25ae304d..b8a1c87b 100644 --- a/tests/util_legal_hold.sh +++ b/tests/util_legal_hold.sh @@ -35,4 +35,25 @@ get_and_check_legal_hold() { return 1 fi return 0 -} \ No newline at end of file +} + +check_legal_hold_without_lock_enabled() { + if [ $# -ne 2 ]; then + log 2 "'check_legal_hold_without_lock_enabled' requires bucket, key names" + return 1 + fi + if get_object_legal_hold_rest "$1" "$2"; then + log 2 "get legal hold using REST succeeded without lock enabled" + return 1 + fi + log 5 "legal hold info: $(cat "$TEST_FILE_FOLDER/object_legal_hold.txt")" + if ! code=$(xmllint --xpath '//*[local-name()="Code"]/text()' "$TEST_FILE_FOLDER/object_legal_hold.txt" 2>&1); then + log 2 "error getting error code: $code" + return 1 + fi + if [ "$code" != "InvalidRequest" ]; then + log 2 "code mismatch (expected 'InvalidRequest', actual '$code')" + return 1 + fi + return 0 +} diff --git a/tests/util_multipart.sh b/tests/util_multipart.sh index 12a6a77b..12274b4f 100644 --- a/tests/util_multipart.sh +++ b/tests/util_multipart.sh @@ -159,6 +159,37 @@ start_multipart_upload_and_list_parts() { export listed_parts } +create_list_check_multipart_uploads() { + if [ $# -ne 3 ]; then + log 2 "list multipart uploads command requires bucket and two keys" + return 1 + fi + if ! create_and_list_multipart_uploads "$1" "$2" "$3"; then + log 2 "error creating and listing multipart uploads" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "Uploads: $uploads" + raw_uploads=$(echo "$uploads" | grep -v "InsecureRequestWarning") + if ! key_one=$(echo "$raw_uploads" | jq -r '.Uploads[0].Key' 2>&1); then + log 2 "error getting key one: $key_one" + return 1 + fi + if ! key_two=$(echo "$raw_uploads" | jq -r '.Uploads[1].Key' 2>&1); then + log 2 "error getting key two: $key_two" + return 1 + fi + if [[ "$2" != "$key_one" ]]; then + log 2 "Key mismatch ($2, $key_one)" + return 1 + fi + if [[ "$3" != "$key_two" ]]; then + log 2 "Key mismatch ($3, $key_two)" + return 1 + fi + return 0 +} + # list unfinished multipart uploads # params: bucket, key one, key two # export current two uploads on success, return 1 for error diff --git a/tests/util_users.sh b/tests/util_users.sh index f14d700f..a2aef047 100644 --- a/tests/util_users.sh +++ b/tests/util_users.sh @@ -118,7 +118,7 @@ cat < "$TEST_FILE_FOLDER"/user_policy_file ] } EOF - if ! error=$(aws iam put-user-policy --user-name "$1" --policy-name "UserPolicy" --policy-document "file://$TEST_FILE_FOLDER/user_policy_file" 2>&1); then + if ! error=$(send_command aws iam put-user-policy --user-name "$1" --policy-name "UserPolicy" --policy-document "file://$TEST_FILE_FOLDER/user_policy_file" 2>&1); then log 2 "error putting user policy: $error" return 1 fi @@ -155,7 +155,7 @@ create_user_direct() { log 2 "create user direct command requires desired username, role, bucket name" return 1 fi - if ! error=$(aws iam create-user --user-name "$1" 2>&1); then + if ! error=$(send_command aws iam create-user --user-name "$1" 2>&1); then log 2 "error creating new user: $error" return 1 fi @@ -163,7 +163,7 @@ create_user_direct() { log 2 "error attaching user policy" return 1 fi - if ! keys=$(aws iam create-access-key --user-name "$1" 2>&1); then + if ! keys=$(send_command aws iam create-access-key --user-name "$1" 2>&1); then log 2 "error creating keys for new user: $keys" return 1 fi @@ -184,7 +184,7 @@ create_user_with_user() { log 2 "create user with user command requires creator ID, key, and new user ID, key, and role" return 1 fi - if ! error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" create-user --access "$3" --secret "$4" --role "$5" 2>&1); then + if ! error=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" create-user --access "$3" --secret "$4" --role "$5" 2>&1); then log 2 "error creating user: $error" return 1 fi @@ -194,7 +194,7 @@ create_user_with_user() { list_users_direct() { log 6 "list_users_direct" # AWS_ENDPOINT_URL of s3.amazonaws.com doesn't work here - if ! users=$(aws --profile="$AWS_PROFILE" iam list-users 2>&1); then + if ! users=$(send_command aws --profile="$AWS_PROFILE" iam list-users 2>&1); then log 2 "error listing users via direct s3 call: $users" return 1 fi @@ -229,7 +229,7 @@ list_users() { list_users_versitygw() { log 6 "list_users_versitygw" - users=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users) || local list_result=$? + users=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users) || local list_result=$? if [[ $list_result -ne 0 ]]; then echo "error listing users: $users" return 1 @@ -267,17 +267,17 @@ delete_user_direct() { log 2 "delete user direct command requires username" return 1 fi - if ! policies=$(aws iam list-user-policies --user-name "$1" --query 'PolicyNames' --output text 2>&1); then + if ! policies=$(send_command aws iam list-user-policies --user-name "$1" --query 'PolicyNames' --output text 2>&1); then log 2 "error getting user policies: $error" return 1 fi for policy_name in $policies; do - if ! user_policy_delete_error=$(aws iam delete-user-policy --user-name "$1" --policy-name "$policy_name" 2>&1); then + if ! user_policy_delete_error=$(send_command aws iam delete-user-policy --user-name "$1" --policy-name "$policy_name" 2>&1); then log 2 "error deleting user policy: $user_policy_delete_error" return 1 fi done - if ! keys=$(aws iam list-access-keys --user-name "$1" 2>&1); then + if ! keys=$(send_command aws iam list-access-keys --user-name "$1" 2>&1); then log 2 "error getting keys: $keys" return 1 fi @@ -286,12 +286,12 @@ delete_user_direct() { return 1 fi if [[ $key != "null" ]]; then - if ! error=$(aws iam delete-access-key --user-name "$1" --access-key-id "$key" 2>&1); then + if ! error=$(send_command aws iam delete-access-key --user-name "$1" --access-key-id "$key" 2>&1); then log 2 "error deleting access key: $error" return 1 fi fi - if ! error=$(aws --profile="$AWS_PROFILE" iam delete-user --user-name "$1" 2>&1); then + if ! error=$(send_command aws --profile="$AWS_PROFILE" iam delete-user --user-name "$1" 2>&1); then log 2 "error deleting user: $error" return 1 fi @@ -305,7 +305,7 @@ delete_user_versitygw() { return 1 fi log 5 "$VERSITY_EXE admin --allow-insecure --access $AWS_ACCESS_KEY_ID --secret $AWS_SECRET_ACCESS_KEY --endpoint-url $AWS_ENDPOINT_URL delete-user --access $1" - if ! error=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" delete-user --access "$1" 2>&1); then + if ! error=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" delete-user --access "$1" 2>&1); then log 2 "error deleting user: $error" export error return 1 @@ -368,7 +368,7 @@ change_bucket_owner() { return 0 fi log 5 "changing owner for bucket $3, new owner: $4" - error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" change-bucket-owner --bucket "$3" --owner "$4" 2>&1) || local change_result=$? + error=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" change-bucket-owner --bucket "$3" --owner "$4" 2>&1) || local change_result=$? if [[ $change_result -ne 0 ]]; then echo "error changing bucket owner: $error" return 1 @@ -382,7 +382,7 @@ get_bucket_owner() { log 2 "'get bucket owner' command requires bucket name" return 1 fi - if ! buckets=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-buckets 2>&1); then + if ! buckets=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-buckets 2>&1); then log 2 "error listing buckets: $buckets" return 1 fi diff --git a/tests/util_versioning.sh b/tests/util_versioning.sh index 08754b6a..acbe2145 100644 --- a/tests/util_versioning.sh +++ b/tests/util_versioning.sh @@ -158,3 +158,47 @@ get_and_check_versions_rest() { done return 0 } + +check_versions_after_file_deletion() { + if [ $# -ne 2 ]; then + log 2 "'check_versions_after_file_deletion' requires bucket, key" + return 1 + fi + if ! list_object_versions_rest "$1"; then + log 2 "error listing object versions" + return 1 + fi + log 5 "versions: $(cat "$TEST_FILE_FOLDER/object_versions.txt")" + if ! version_key=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/object_versions.txt" 2>&1); then + log 2 "error getting Version 'Key' value: $version_key" + return 1 + fi + if [ "$version_key" != "$2" ]; then + log 2 "version key mismatch (expected $2, actual $version_key)" + return 1 + fi + if ! version_id=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="VersionId"]/text()' "$TEST_FILE_FOLDER/object_versions.txt" 2>&1); then + log 2 "error getting Version 'VersionID' value: $version_id" + return 1 + fi + if [ "$version_id" != "null" ]; then + log 2 "version ID mismatch (expected 'null', actual '$version_id')" + return 1 + fi + if ! marker_key=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/object_versions.txt" 2>&1); then + log 2 "error getting Version 'Key' value: $marker_key" + return 1 + fi + if [ "$marker_key" != "$2" ]; then + log 2 "delete marker key mismatch (expected $2, actual $marker_key)" + return 1 + fi + if ! marker_id=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="VersionId"]/text()' "$TEST_FILE_FOLDER/object_versions.txt" 2>&1); then + log 2 "error getting Version 'VersionID' value: $versioning_info" + return 1 + fi + if [ "$marker_id" == "null" ]; then + log 2 "delete marker ID expected to be non-null" + return 1 + fi +} diff --git a/tests/versity.sh b/tests/versity.sh index 852366dc..60fbe653 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -27,6 +27,11 @@ start_versity_process() { fi IFS=' ' read -r -a full_command <<< "${base_command[@]}" log 5 "versity command: ${full_command[*]}" + if [ -n "$COMMAND_LOG" ]; then + mask_args "${full_command[*]}" + # shellcheck disable=SC2154 + echo "${masked_args[@]}" >> "$COMMAND_LOG" + fi if [ -n "$VERSITY_LOG_FILE" ]; then "${full_command[@]}" >> "$VERSITY_LOG_FILE" 2>&1 & else