From e81fac9558d7917326da3f069194ec965f30bc70 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Fri, 28 Jun 2024 15:02:21 -0300 Subject: [PATCH] test: multipart abort policy, direct s3 user setup, ACLs on/off command --- tests/commands/abort_multipart_upload.sh | 19 +- tests/commands/create_multipart_upload.sh | 32 ++- tests/commands/get_bucket_acl.sh | 13 + .../commands/get_bucket_ownership_controls.sh | 36 +++ tests/commands/put_bucket_acl.sh | 16 +- tests/env.sh | 10 +- tests/test_aws.sh | 244 ++++++++++-------- tests/test_user_common.sh | 99 +++---- tests/util.sh | 31 ++- tests/util_users.sh | 216 ++++++++++++++-- 10 files changed, 502 insertions(+), 214 deletions(-) create mode 100644 tests/commands/get_bucket_ownership_controls.sh diff --git a/tests/commands/abort_multipart_upload.sh b/tests/commands/abort_multipart_upload.sh index e9fbdb71..dde8a4bb 100644 --- a/tests/commands/abort_multipart_upload.sh +++ b/tests/commands/abort_multipart_upload.sh @@ -2,13 +2,24 @@ abort_multipart_upload() { if [ $# -ne 3 ]; then - echo "command to run abort requires bucket, key, upload ID" + log 2 "'abort multipart upload' command requires bucket, key, upload ID" return 1 fi + if ! error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then + log 2 "Error aborting upload: $error" + return 1 + fi + return 0 +} - error=$(aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3") || local aborted=$? - if [[ $aborted -ne 0 ]]; then - echo "Error aborting upload: $error" +abort_multipart_upload_with_user() { + if [ $# -ne 5 ]; then + log 2 "'abort multipart upload' command requires bucket, key, upload ID, username, password" + return 1 + fi + if ! abort_multipart_upload_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$2" --upload-id "$3" 2>&1); then + log 2 "Error aborting upload: $abort_multipart_upload_error" + export abort_multipart_upload_error return 1 fi return 0 diff --git a/tests/commands/create_multipart_upload.sh b/tests/commands/create_multipart_upload.sh index c5e68c4c..4e1a2759 100644 --- a/tests/commands/create_multipart_upload.sh +++ b/tests/commands/create_multipart_upload.sh @@ -9,16 +9,38 @@ create_multipart_upload() { return 1 fi - local multipart_data - multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2") || local created=$? - if [[ $created -ne 0 ]]; then - log 2 "Error creating multipart upload: $upload_id" + if ! multipart_data=$(aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then + log 2 "Error creating multipart upload: $multipart_data" return 1 fi - upload_id=$(echo "$multipart_data" | jq '.UploadId') + if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then + log 2 "error parsing upload ID: $upload_id" + return 1 + fi upload_id="${upload_id//\"/}" export upload_id + return 0 +} + +create_multipart_upload_with_user() { + if [ $# -ne 4 ]; then + log 2 "create multipart upload function must have bucket, key, username, password" + return 1 + fi + + if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then + log 2 "Error creating multipart upload: $multipart_data" + return 1 + fi + + if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then + log 2 "error parsing upload ID: $upload_id" + return 1 + fi + upload_id="${upload_id//\"/}" + export upload_id + return 0 } create_multipart_upload_params() { diff --git a/tests/commands/get_bucket_acl.sh b/tests/commands/get_bucket_acl.sh index 4ea74fa7..b6b52513 100644 --- a/tests/commands/get_bucket_acl.sh +++ b/tests/commands/get_bucket_acl.sh @@ -19,4 +19,17 @@ get_bucket_acl() { return 1 fi export acl +} + +get_bucket_acl_with_user() { + if [ $# -ne 3 ]; then + log 2 "'get bucket ACL with user' command requires bucket name, username, password" + return 1 + fi + if ! bucket_acl=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-acl --bucket "$1" 2>&1); then + log 2 "error getting bucket ACLs: $bucket_acl" + return 1 + fi + export bucket_acl + return 0 } \ No newline at end of file diff --git a/tests/commands/get_bucket_ownership_controls.sh b/tests/commands/get_bucket_ownership_controls.sh new file mode 100644 index 00000000..3c9ca0d5 --- /dev/null +++ b/tests/commands/get_bucket_ownership_controls.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +get_bucket_ownership_controls() { + if [[ $# -ne 1 ]]; then + log 2 "'get bucket ownership controls' command requires bucket name" + return 1 + fi + + if ! raw_bucket_ownership_controls=$(aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then + log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls" + return 1 + fi + + log 5 "Raw bucket Ownership Controls: $raw_bucket_ownership_controls" + bucket_ownership_controls=$(echo "$raw_bucket_ownership_controls" | grep -v "InsecureRequestWarning") + export bucket_ownership_controls + return 0 +} + +get_object_ownership_rule() { + if [[ $# -ne 1 ]]; then + log 2 "'get object ownership rule' command requires bucket name" + return 1 + fi + if ! get_bucket_ownership_controls "$1"; then + log 2 "error getting bucket ownership controls" + return 1 + fi + if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then + log 2 "error getting object ownership rule: $object_ownership_rule" + return 1 + fi + log 5 "object ownership rule: $object_ownership_rule" + export object_ownership_rule + return 0 +} \ No newline at end of file diff --git a/tests/commands/put_bucket_acl.sh b/tests/commands/put_bucket_acl.sh index 6db0da57..2340f3ba 100644 --- a/tests/commands/put_bucket_acl.sh +++ b/tests/commands/put_bucket_acl.sh @@ -29,8 +29,20 @@ put_bucket_canned_acl() { return 1 fi if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2"); then - log 2 "error resetting bucket acls: $error" + log 2 "error re-setting bucket acls: $error" return 1 fi return 0 -} \ No newline at end of file +} + +put_bucket_canned_acl_with_user() { + if [[ $# -ne 2 ]]; then + log 2 "'put bucket canned acl with user' command requires bucket name, canned ACL, username, password" + return 1 + fi + if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2"); then + log 2 "error re-setting bucket acls: $error" + return 1 + fi + return 0 +} diff --git a/tests/env.sh b/tests/env.sh index 46ce89d4..4fadedea 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -56,10 +56,13 @@ check_universal_vars() { elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then log 2 "No AWS secret access key set" return 1 + elif [ -z "$AWS_REGION" ]; then + log 2 "No AWS region set" + return 1 elif [ -z "$AWS_PROFILE" ]; then log 2 "No AWS profile set" return 1 - elif [ -z "$AWS_ENDPOINT_URL" ]; then + elif [ "$DIRECT" != "true" ] && [ -z "$AWS_ENDPOINT_URL" ]; then log 2 "No AWS endpoint URL set" return 1 elif [[ $RUN_VERSITYGW != "true" ]] && [[ $RUN_VERSITYGW != "false" ]]; then @@ -78,7 +81,7 @@ check_universal_vars() { log 2 "RECREATE_BUCKETS must be 'true' or 'false'" return 1 fi - export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_PROFILE AWS_ENDPOINT_URL RUN_VERSITYGW \ + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL RUN_VERSITYGW \ BUCKET_ONE_NAME BUCKET_TWO_NAME RECREATE_BUCKETS if [[ -n "$TEST_LOG_FILE" ]]; then export TEST_LOG_FILE @@ -86,6 +89,9 @@ check_universal_vars() { if [[ -n "$VERSITY_LOG_FILE" ]]; then export VERSITY_LOG_FILE fi + if [[ -n "$DIRECT" ]]; then + export DIRECT + fi } check_versity_vars() { diff --git a/tests/test_aws.sh b/tests/test_aws.sh index 58c2eb77..7aef6985 100755 --- a/tests/test_aws.sh +++ b/tests/test_aws.sh @@ -681,24 +681,20 @@ legal_hold_retention_setup() { @test "test-list-objects-delimiter" { folder_name="two" object_name="three" - create_test_folder "$folder_name" || local created=$? - [[ $created -eq 0 ]] || fail "error creating folder" - create_test_files "$folder_name"/"$object_name" || created=$? - [[ $created -eq 0 ]] || fail "error creating file" + create_test_folder "$folder_name" || fail "error creating folder" + create_test_files "$folder_name"/"$object_name" || fail "error creating file" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket" - put_object "aws" "$test_file_folder/$folder_name/$object_name" "$BUCKET_ONE_NAME" "$folder_name/$object_name" || local copy_result=$? - [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" + put_object "aws" "$test_file_folder/$folder_name/$object_name" "$BUCKET_ONE_NAME" "$folder_name/$object_name" || fail "failed to add object to bucket" list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/" - prefix=$(echo "${objects[@]}" | jq ".CommonPrefixes[0].Prefix") - [[ $prefix == "\""$folder_name/"\"" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)" + prefix=$(echo "${objects[@]}" | jq -r ".CommonPrefixes[0].Prefix" 2>&1) || fail "error getting object prefix from object list: $prefix" + [[ $prefix == "$folder_name/" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)" list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#" - key=$(echo "${objects[@]}" | jq ".Contents[0].Key") - [[ $key == "\""$folder_name/$object_name"\"" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)" + key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1) || fail "error getting key from object list: $key" + [[ $key == "$folder_name/$object_name" ]] || fail "key doesn't match (expected $key, actual $folder_name/$object_name)" delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" delete_test_files $folder_name @@ -791,10 +787,6 @@ EOF action="s3:GetObject" resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - setup_policy_with_single_statement "$test_file_folder/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" @@ -805,32 +797,7 @@ EOF check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" || fail "policy not empty after deletion" fi - if put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file"; then - fail "put succeeded despite invalid username" - fi - [[ "$put_bucket_policy_error" == *"MalformedPolicy"* ]] || fail "invalid policy error: $put_bucket_policy_error" - -# cat < "$test_file_folder"/acl_file -#{ -# "Grants": [ -# { -# "Grantee": { -# "ID": "ABCDEFG", -# "Type": "CanonicalUser" -# }, -# "Permission": "READ" -# } -# ], -# "Owner": { -# "ID": "$AWS_ACCESS_KEY_ID" -# } -#} -#EOF -# - #put_bucket_acl "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/acl_file" || fail "error putting acl" - #put_bucket_canned_acl "$BUCKET_ONE_NAME" "public-read-write" || fail "error putting acl" - - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password"; then fail "get object with user succeeded despite lack of permissions" fi @@ -838,8 +805,6 @@ EOF [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" - #get_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "error getting bucket policy" - #log 5 "$bucket_policy" get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password" || fail "error getting object after permissions" compare_files "$test_file_folder/$test_file" "$test_file_folder/$test_file-copy" || fail "files not equal" delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" @@ -866,10 +831,7 @@ EOF action="s3:GetObject" resource="arn:aws:s3:::$BUCKET_ONE_NAME/test_file" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" @@ -907,10 +869,7 @@ EOF action="s3:GetObject" resource="arn:aws:s3:::$BUCKET_ONE_NAME/policy_file*" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user account" + setup_user "$username" "$password" "user" || fail "error creating user account" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" @@ -950,10 +909,7 @@ EOF action="s3:GetObject" resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" @@ -997,10 +953,7 @@ EOF } EOF - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" put_object "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object to bucket" @@ -1047,10 +1000,7 @@ EOF } EOF - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" log 5 "Policy: $(cat "$test_file_folder/$policy_file")" @@ -1087,10 +1037,7 @@ EOF action="s3:PutObject" resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" log 5 "Policy: $(cat "$test_file_folder/$policy_file")" @@ -1130,10 +1077,7 @@ EOF action="s3:DeleteObject" resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file_two" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" @@ -1167,10 +1111,7 @@ EOF action="s3:GetBucketPolicy" resource="arn:aws:s3:::$BUCKET_ONE_NAME" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" @@ -1207,10 +1148,7 @@ EOF action="s3:ListBucketMultipartUploads" resource="arn:aws:s3:::$BUCKET_ONE_NAME" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" @@ -1246,10 +1184,7 @@ EOF action="s3:PutBucketPolicy" resource="arn:aws:s3:::$BUCKET_ONE_NAME" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" @@ -1286,10 +1221,7 @@ EOF action="s3:DeleteBucketPolicy" resource="arn:aws:s3:::$BUCKET_ONE_NAME" - if user_exists "$username"; then - delete_user "$username" || fail "failed to delete user '$username'" - fi - create_user "$username" "$password" "user" || fail "error creating user" + setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" if delete_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then @@ -1302,6 +1234,34 @@ EOF delete_test_files "$policy_file" } +@test "test_policy_get_bucket_acl" { + # TODO (https://github.com/versity/versitygw/issues/637) + if [[ $RECREATE_BUCKETS == "false" ]]; then + return 0 + fi + policy_file="policy_file" + username="ABCDEFG" + password="HIJKLMN" + + create_test_files "$policy_file" || fail "error creating policy file, test files" + + effect="Allow" + principal="$username" + action="s3:GetBucketAcl" + resource="arn:aws:s3:::$BUCKET_ONE_NAME" + + setup_user "$username" "$password" "user" || fail "error creating user" + + setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + #put_bucket_canned_acl "$BUCKET_ONE_NAME" "private" || fail "error putting bucket canned ACL" + if get_bucket_acl_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then + fail "user able to get bucket ACLs despite permissions" + fi + setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" + put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" + get_bucket_acl_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "error getting bucket ACL despite permissions" +} + # ensure that lists of files greater than a size of 1000 (pagination) are returned properly #@test "test_list_objects_file_count" { # test_common_list_objects_file_count "aws" @@ -1332,15 +1292,18 @@ EOF #} @test "test_head_bucket" { - setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - head_bucket "aws" "$BUCKET_ONE_NAME" + setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info" + log 5 "INFO: $bucket_info" + region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region" + [[ $region != "" ]] || fail "empty bucket region" delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" } @test "test_head_bucket_invalid_name" { - head_bucket "aws" "" || local head_result=$? - [[ $head_result -ne 0 ]] || fail "able to get bucket info for invalid name" + if head_bucket "aws" ""; then + fail "able to get bucket info for invalid name" + fi } @test "test_head_bucket_doesnt_exist" { @@ -1357,23 +1320,90 @@ EOF test_key="x-test-data" test_value="test-value" - create_test_files "$object_one" || local created=$? - [[ $created -eq 0 ]] || fail "Error creating test files" + create_test_files "$object_one" || fail "error creating test files" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket" object="$test_file_folder"/"$object_one" - put_object_with_metadata "aws" "$object" "$BUCKET_ONE_NAME" "$object_one" "$test_key" "$test_value" || copy_result=$? - [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" - object_exists "aws" "$BUCKET_ONE_NAME" "$object_one" || local exists_result_one=$? - [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket" + put_object_with_metadata "aws" "$object" "$BUCKET_ONE_NAME" "$object_one" "$test_key" "$test_value" || fail "failed to add object to bucket" + object_exists "aws" "$BUCKET_ONE_NAME" "$object_one" || fail "object not found after being added to bucket" - get_object_metadata "aws" "$BUCKET_ONE_NAME" "$object_one" || get_result=$? - [[ $get_result -eq 0 ]] || fail "error getting object metadata" - key=$(echo "$metadata" | jq 'keys[]') - value=$(echo "$metadata" | jq '.[]') - [[ $key == "\"$test_key\"" ]] || fail "keys doesn't match (expected $key, actual \"$test_key\")" - [[ $value == "\"$test_value\"" ]] || fail "values doesn't match (expected $value, actual \"$test_value\")" + get_object_metadata "aws" "$BUCKET_ONE_NAME" "$object_one" || fail "error getting object metadata" + key=$(echo "$metadata" | jq -r 'keys[]' 2>&1) || fail "error getting key from metadata: $key" + value=$(echo "$metadata" | jq -r '.[]' 2>&1) || fail "error getting value from metadata: $value" + [[ $key == "$test_key" ]] || fail "keys doesn't match (expected $key, actual \"$test_key\")" + [[ $value == "$test_value" ]] || fail "values doesn't match (expected $value, actual \"$test_value\")" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files "$object_one" } +@test "test_policy_abort_multipart_upload" { + policy_file="policy_file" + test_file="test_file" + username="ABCDEFG" + + create_test_files "$policy_file" || fail "error creating policy file" + create_large_file "$test_file" || fail "error creating large file" + setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + if [[ $DIRECT == "true" ]]; then + setup_user_direct "$username" "user" "$BUCKET_ONE_NAME" || fail "error setting up direct user $username" + principal="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/$username\"}" + # shellcheck disable=SC2154 + username=$key_id + # shellcheck disable=SC2154 + password=$secret_key + else + password="HIJLKMN" + setup_user "$username" "$password" "user" || fail "error setting up user $username" + principal="\"$username\"" + fi + + cat < "$test_file_folder"/$policy_file +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": $principal, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*" + }, + { + "Effect": "Deny", + "Principal": $principal, + "Action": "s3:AbortMultipartUpload", + "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*" + } + ] +} +EOF + put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting first policy" + + create_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$username" "$password" || fail "error creating multipart upload" + if abort_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$upload_id" "$username" "$password"; then + fail "abort multipart upload succeeded despite lack of permissions" + fi + # shellcheck disable=SC2154 + [[ "$abort_multipart_upload_error" == *"AccessDenied"* ]] || fail "unexpected abort error: $abort_multipart_upload_error" + + cat < "$test_file_folder"/$policy_file +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": $principal, + "Action": "s3:AbortMultipartUpload", + "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*" + } + ] +} +EOF + + put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" + abort_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$upload_id" "$username" "$password" || fail "error aborting multipart upload despite permissions" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files "$policy_file" "$test_file" +} diff --git a/tests/test_user_common.sh b/tests/test_user_common.sh index 9dcc61f5..0f0de7bb 100755 --- a/tests/test_user_common.sh +++ b/tests/test_user_common.sh @@ -15,31 +15,21 @@ test_admin_user() { admin_password="123456" user_password="789012" - user_exists "$admin_username" || local admin_exists_result=$? - if [[ $admin_exists_result -eq 0 ]]; then - delete_user "$admin_username" || local delete_admin_result=$? - [[ $delete_admin_result -eq 0 ]] || fail "failed to delete admin user" - fi - create_user "$admin_username" "$admin_password" "admin" || create_admin_result=$? - [[ $create_admin_result -eq 0 ]] || fail "failed to create admin user" + setup_user "$admin_username" "$admin_password" "admin" || fail "error setting up admin user" - user_exists "$user_username" || local user_exists_result=$? - if [[ $user_exists_result -eq 0 ]]; then - delete_user "$user_username" || local delete_user_result=$? - [[ $delete_user_result -eq 0 ]] || fail "failed to delete user user" + if user_exists "$user_username"; then + delete_user "$user_username" || fail "failed to delete user '$user_username'" fi - create_user_with_user "$admin_username" "$admin_password" "$user_username" "$user_password" "user" + create_user_with_user "$admin_username" "$admin_password" "$user_username" "$user_password" "user" || fail "failed to create user '$user_username'" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - delete_bucket "aws" "versity-gwtest-admin-bucket" || local delete_result=$? - [[ $delete_result -eq 0 ]] || fail "error deleting bucket if it exists" - create_bucket_with_user "aws" "versity-gwtest-admin-bucket" "$admin_username" "$admin_password" || create_result_two=$? - [[ $create_result_two -eq 0 ]] || fail "error creating bucket with user" + setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + delete_bucket "aws" "versity-gwtest-admin-bucket" || fail "error deleting bucket if it exists" + create_bucket_with_user "aws" "versity-gwtest-admin-bucket" "$admin_username" "$admin_password" || fail "error creating bucket with admin user" bucket_one_found=false bucket_two_found=false - list_buckets_with_user "aws" "$admin_username" "$admin_password" + list_buckets_with_user "aws" "$admin_username" "$admin_password" || fail "error listing buckets with admin user" + # shellcheck disable=SC2154 for bucket in "${bucket_array[@]}"; do if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then bucket_one_found=true @@ -53,8 +43,7 @@ test_admin_user() { if [ $bucket_one_found == false ] || [ $bucket_two_found == false ]; then fail "not all expected buckets listed" fi - change_bucket_owner "$admin_username" "$admin_password" "versity-gwtest-admin-bucket" "$user_username" || local change_result=$? - [[ $change_result -eq 0 ]] || fail "error changing bucket owner" + change_bucket_owner "$admin_username" "$admin_password" "versity-gwtest-admin-bucket" "$user_username" || fail "error changing bucket owner" delete_bucket "aws" "versity-gwtest-admin-bucket" delete_user "$user_username" @@ -69,17 +58,11 @@ test_create_user_already_exists() { username="ABCDEG" password="123456" - user_exists "$username" || local exists_result=$? - if [[ $exists_result -eq 0 ]]; then - delete_user "$username" || local delete_result=$? - [[ $delete_result -eq 0 ]] || fail "failed to delete user '$username'" + setup_user "$username" "123456" "admin" || fail "error setting up user" + if create_user "$username" "123456" "admin"; then + fail "'user already exists' error not returned" fi - create_user "$username" "123456" "admin" || local create_result=$? - [[ $create_result -eq 0 ]] || fail "error creating user" - create_user "$username" "123456" "admin" || local create_result=$? - [[ $create_result -eq 1 ]] || fail "'user already exists' error not returned" - delete_bucket "aws" "versity-gwtest-admin-bucket" delete_user "$username" } @@ -92,31 +75,24 @@ test_user_user() { username="ABCDEG" password="123456" - user_exists "$username" || local exists_result=$? - if [[ $exists_result -eq 0 ]]; then - delete_user "$username" || local delete_result=$? - [[ $delete_result -eq 0 ]] || fail "failed to delete user '$username'" - fi + setup_user "$username" "$password" "user" || fail "error setting up user" delete_bucket "aws" "versity-gwtest-user-bucket" + setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket '$BUCKET_ONE_NAME'" - create_user "$username" "123456" "user" || local create_result=$? - [[ $create_result -eq 0 ]] || fail "error creating user" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - - create_bucket_with_user "aws" "versity-gwtest-user-bucket" "$username" "$password" || create_result_two=$? - [[ $create_result_two -eq 1 ]] || fail "creating bucket with 'user' account failed to return error" + if create_bucket_with_user "aws" "versity-gwtest-user-bucket" "$username" "$password"; then + fail "creating bucket with 'user' account failed to return error" + fi + # shellcheck disable=SC2154 [[ $error == *"Access Denied"* ]] || fail "error message '$error' doesn't contain 'Access Denied'" - create_bucket "aws" "versity-gwtest-user-bucket" || create_result_three=$? - [[ $create_result_three -eq 0 ]] || fail "creating bucket account returned error" + create_bucket "aws" "versity-gwtest-user-bucket" || fail "error creating bucket" - change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "versity-gwtest-user-bucket" "$username" || local change_result=$? - [[ $change_result -eq 0 ]] || fail "error changing bucket owner" - change_bucket_owner "$username" "$password" "versity-gwtest-user-bucket" "admin" || local change_result_two=$? - [[ $change_result_two -eq 1 ]] || fail "user shouldn't be able to change bucket owner" + change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "versity-gwtest-user-bucket" "$username" || fail "error changing bucket owner" + if change_bucket_owner "$username" "$password" "versity-gwtest-user-bucket" "admin"; then + fail "user shouldn't be able to change bucket owner" + fi - list_buckets_with_user "aws" "$username" "$password" + list_buckets_with_user "aws" "$username" "$password" || fail "error listing buckets with user '$username'" bucket_found=false for bucket in "${bucket_array[@]}"; do if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then @@ -141,22 +117,13 @@ test_userplus_operation() { username="ABCDEG" password="123456" - user_exists "$username" || local exists_result=$? - if [[ $exists_result -eq 0 ]]; then - delete_user "$username" || local delete_result=$? - [[ $delete_result -eq 0 ]] || fail "failed to delete user '$username'" - fi delete_bucket "aws" "versity-gwtest-userplus-bucket" + setup_user "$username" "$password" "userplus" || fail "error creating user '$username'" + setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket '$BUCKET_ONE_NAME'" - create_user "$username" "123456" "userplus" || local create_result=$? - [[ $create_result -eq 0 ]] || fail "error creating user" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + create_bucket_with_user "aws" "versity-gwtest-userplus-bucket" "$username" "$password" || fail "error creating bucket with user '$username'" - create_bucket_with_user "aws" "versity-gwtest-userplus-bucket" "$username" "$password" || create_result_two=$? - [[ $create_result_two -eq 0 ]] || fail "error creating bucket" - - list_buckets_with_user "aws" "$username" "$password" + list_buckets_with_user "aws" "$username" "$password" || fail "error listing buckets with user '$username'" bucket_found=false for bucket in "${bucket_array[@]}"; do if [ "$bucket" == "$BUCKET_ONE_NAME" ]; then @@ -169,10 +136,10 @@ test_userplus_operation() { fail "userplus-owned bucket not found in user list" fi - change_bucket_owner "$username" "$password" "versity-gwtest-userplus-bucket" "admin" || local change_result_two=$? - [[ $change_result_two -eq 1 ]] || fail "userplus shouldn't be able to change bucket owner" + if change_bucket_owner "$username" "$password" "versity-gwtest-userplus-bucket" "admin"; then + fail "userplus shouldn't be able to change bucket owner" + fi delete_bucket "aws" "versity-gwtest-admin-bucket" - delete_user "$username" || delete_result=$? - [[ $delete_result -eq 0 ]] || fail "error deleting user" + delete_user "$username" } \ No newline at end of file diff --git a/tests/util.sh b/tests/util.sh index 7f7aeb9c..8c6e519e 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -10,6 +10,7 @@ source ./tests/commands/create_bucket.sh source ./tests/commands/delete_bucket.sh source ./tests/commands/delete_bucket_policy.sh source ./tests/commands/delete_object.sh +source ./tests/commands/get_bucket_ownership_controls.sh source ./tests/commands/get_bucket_tagging.sh source ./tests/commands/get_object_tagging.sh source ./tests/commands/head_bucket.sh @@ -158,7 +159,12 @@ delete_bucket_or_contents() { log 2 "error deleting bucket policies" return 1 fi - if ! put_bucket_canned_acl "$2" "private"; then + if ! get_object_ownership_rule "$2"; then + log 2 "error getting object ownership rule" + return 1 + fi + # shellcheck disable=SC2154 + if [[ "$object_ownership_rule" != "BucketOwnerEnforced" ]] && ! put_bucket_canned_acl "$2" "private"; then log 2 "error resetting bucket ACLs" return 1 fi @@ -656,14 +662,12 @@ multipart_upload_before_completion() { return 1 fi - split_file "$3" "$4" || split_result=$? - if [[ $split_result -ne 0 ]]; then + if ! split_file "$3" "$4"; then log 2 "error splitting file" return 1 fi - create_multipart_upload "$1" "$2" || create_result=$? - if [[ $create_result -ne 0 ]]; then + if ! create_multipart_upload "$1" "$2"; then log 2 "error creating multpart upload" return 1 fi @@ -671,8 +675,7 @@ multipart_upload_before_completion() { parts="[" for ((i = 1; i <= $4; i++)); do # shellcheck disable=SC2154 - upload_part "$1" "$2" "$upload_id" "$3" "$i" || local upload_result=$? - if [[ $upload_result -ne 0 ]]; then + if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then echo "error uploading part $i" return 1 fi @@ -830,18 +833,20 @@ multipart_upload_with_params() { # return 0 for success, 1 for failure run_then_abort_multipart_upload() { if [ $# -ne 4 ]; then - echo "run then abort multipart upload command missing bucket, key, file, and/or part count" + log 2 "run then abort multipart upload command missing bucket, key, file, and/or part count" return 1 fi - multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$? - if [[ $result -ne 0 ]]; then - echo "error performing pre-completion multipart upload" + if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then + log 2 "error performing pre-completion multipart upload" return 1 fi - abort_multipart_upload "$1" "$2" "$upload_id" - return $? + if ! abort_multipart_upload "$1" "$2" "$upload_id"; then + log 2 "error aborting multipart upload" + return 1 + fi + return 0 } # copy a file to/from S3 diff --git a/tests/util_users.sh b/tests/util_users.sh index c2a37f36..b6480286 100644 --- a/tests/util_users.sh +++ b/tests/util_users.sh @@ -1,6 +1,42 @@ #!/usr/bin/env bash -create_user() { +setup_user() { + if [[ $# -ne 3 ]]; then + log 2 "'setup user' command requires username, password, and role" + return 1 + fi + if user_exists "$1"; then + if ! delete_user "$1"; then + log 2 "error deleting user '$1'" + return 1 + fi + fi + if ! create_user_versitygw "$1" "$2" "$3"; then + log 2 "error creating user '$1'" + return 1 + fi + return 0 +} + +setup_user_direct() { + if [[ $# -ne 3 ]]; then + log 2 "'setup user direct' command requires username, role, and bucket" + return 1 + fi + if user_exists "$1"; then + if ! delete_user "$1"; then + log 2 "error deleting user '$1'" + return 1 + fi + fi + if ! create_user_direct "$1" "$2" "$3"; then + log 2 "error creating user" + return 1 + fi + return 0 +} + +create_user_versitygw() { if [[ $# -ne 3 ]]; then log 2 "create user command requires user ID, key, and role" return 1 @@ -25,6 +61,70 @@ create_user_if_nonexistent() { return $? } +put_user_policy() { + if [[ $# -ne 3 ]]; then + log 2 "attaching user policy requires user ID, role, bucket name" + return 1 + fi + if [[ -z "$test_file_folder" ]]; then + log 2 "no test folder defined" + return 1 + fi + + # TODO add other roles + if [[ $2 != "user" ]]; then + log 2 "role for '$2' not currently supported" + return 1 + fi + +cat < "$test_file_folder"/user_policy_file +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "*", + "Resource": "arn:aws:s3:::$3/*" + } + ] +} +EOF + + if ! error=$(aws iam put-user-policy --user-name "$1" --policy-name "UserPolicy" --policy-document "file://$test_file_folder/user_policy_file" 2>&1); then + log 2 "error putting user policy: $error" + return 1 + fi + return 0 +} + +create_user_direct() { + if [[ $# -ne 3 ]]; then + log 2 "create user direct command requires desired username, role, bucket name" + return 1 + fi + if ! error=$(aws iam create-user --user-name "$1" 2>&1); then + log 2 "error creating new user: $error" + return 1 + fi + if ! put_user_policy "$1" "$2" "$3"; then + log 2 "error attaching user policy" + return 1 + fi + if ! keys=$(aws iam create-access-key --user-name "$1" 2>&1); then + log 2 "error creating keys for new user: $keys" + return 1 + fi + key_id=$(echo "$keys" | jq -r ".AccessKey.AccessKeyId") + export key_id + secret_key=$(echo "$keys" | jq -r ".AccessKey.SecretAccessKey") + export secret_key + + # propagation delay occurs when user is added to IAM, so wait a few seconds + sleep 5 + + return 0 +} + create_user_with_user() { if [[ $# -ne 5 ]]; then log 2 "create user with user command requires creator ID, key, and new user ID, key, and role" @@ -37,7 +137,41 @@ create_user_with_user() { return 0 } +list_users_direct() { + # AWS_ENDPOINT_URL of s3.amazonaws.com doesn't work here + if ! users=$(aws --profile="$AWS_PROFILE" iam list-users 2>&1); then + log 2 "error listing users via direct s3 call: $users" + return 1 + fi + parsed_users=() + if ! users_list=$(echo "$users" | jq -r ".Users[].UserName" 2>&1); then + log 2 "error parsing users array: $users_list" + return 1 + fi + while IFS= read -r line; do + parsed_users+=("$line") + done <<< "$users_list" + log 5 "parsed users: ${parsed_users[*]}" + export parsed_users + return 0 +} + list_users() { + if [[ $DIRECT == "true" ]]; then + if ! list_users_direct; then + log 2 "error listing users via direct s3 call" + return 1 + fi + return 0 + fi + if ! list_users_versitygw; then + log 2 "error listing versitygw users" + return 1 + fi + return 0 +} + +list_users_versitygw() { users=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users) || local list_result=$? if [[ $list_result -ne 0 ]]; then echo "error listing users: $users" @@ -53,15 +187,15 @@ list_users() { user_exists() { if [[ $# -ne 1 ]]; then - echo "user exists command requires username" + log 2 "user exists command requires username" return 2 fi - list_users || local list_result=$? - if [[ $list_result -ne 0 ]]; then - echo "error listing user" + if ! list_users; then + log 2 "error listing user" return 2 fi for element in "${parsed_users[@]}"; do + log 5 "user: $element" if [[ $element == "$1" ]]; then return 0 fi @@ -69,21 +203,73 @@ user_exists() { return 1 } +delete_user_direct() { + if [[ $# -ne 1 ]]; then + log 2 "delete user direct command requires username" + return 1 + fi + if ! policies=$(aws iam list-user-policies --user-name "$1" --query 'PolicyNames' --output text 2>&1); then + log 2 "error getting user policies: $error" + return 1 + fi + for policy_name in $policies; do + if ! user_policy_delete_error=$(aws iam delete-user-policy --user-name "$1" --policy-name "$policy_name" 2>&1); then + log 2 "error deleting user policy: $user_policy_delete_error" + return 1 + fi + done + if ! keys=$(aws iam list-access-keys --user-name "$1" 2>&1); then + log 2 "error getting keys: $keys" + return 1 + fi + if ! key=$(echo "$keys" | jq -r ".AccessKeyMetadata[0].AccessKeyId" 2>&1); then + log 2 "error getting key ID: $key" + return 1 + fi + if [[ $key != "null" ]]; then + if ! error=$(aws iam delete-access-key --user-name "$1" --access-key-id "$key" 2>&1); then + log 2 "error deleting access key: $error" + return 1 + fi + fi + if ! error=$(aws --profile="$AWS_PROFILE" iam delete-user --user-name "$1" 2>&1); then + log 2 "error deleting user: $error" + return 1 + fi + return 0 +} + +delete_user_versitygw() { + if [[ $# -ne 1 ]]; then + log 2 "delete user via versitygw command requires user ID or username" + return 1 + fi + log 5 "$VERSITY_EXE admin --allow-insecure --access $AWS_ACCESS_KEY_ID --secret $AWS_SECRET_ACCESS_KEY --endpoint-url $AWS_ENDPOINT_URL delete-user --access $1" + if ! error=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" delete-user --access "$1" 2>&1); then + log 2 "error deleting user: $error" + export error + return 1 + fi + return 0 +} + delete_user() { if [[ $# -ne 1 ]]; then - echo "delete user command requires user ID" - return 1 - fi - log 5 "$VERSITY_EXE admin --allow-insecure --access $AWS_ACCESS_KEY_ID --secret $AWS_SECRET_ACCESS_KEY --endpoint-url $AWS_ENDPOINT_URL delete-user --access $1" - error=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" delete-user --access "$1") || local delete_result=$? - - - if [[ $delete_result -ne 0 ]]; then - echo "error deleting user: $error" - export error + log 2 "delete user command requires user ID" + return 1 + fi + if [[ $DIRECT == "true" ]]; then + if ! delete_user_direct "$1"; then + log 2 "error deleting user direct via s3" return 1 fi + log 5 "user '$1' deleted successfully" return 0 + fi + if ! delete_user_versitygw "$1"; then + log 2 "error deleting user via versitygw" + return 1 + fi } change_bucket_owner() {