diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index d1a5dae5..2d1ff7c0 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -141,6 +141,7 @@ jobs: PASSWORD_ONE: 1234567 USERNAME_TWO: HIJKLMN PASSWORD_TWO: 8901234 + TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files run: | make testbin export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST diff --git a/tests/.env.default b/tests/.env.default index bb689bfb..a223cd0a 100644 --- a/tests/.env.default +++ b/tests/.env.default @@ -25,3 +25,4 @@ USERNAME_ONE=ABCDEFG PASSWORD_ONE=HIJKLMN USERNAME_TWO=HIJKLMN PASSWORD_TWO=OPQRSTU +TEST_FILE_FOLDER=$PWD/versity-gwtest-files diff --git a/tests/README.md b/tests/README.md index dc085d27..1c05ba95 100644 --- a/tests/README.md +++ b/tests/README.md @@ -10,9 +10,9 @@ * **s3cmd**: Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md). * **mc**: Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html). 3. Install **BATS**. Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html). -4. Install **bats-support** and **bats-assert**. This can be done by saving the root folder of each repo (both located in https://github.com/bats-core/) in the `tests` folder. +4. Install **bats-support** and **bats-assert**. This can be done by saving the root folder of each repo (https://github.com/bats-core/bats-support and https://github.com/ztombol/bats-assert) in the `tests` folder. 5. If running on Mac OS, install **jq** with the command `brew install jq`. -6. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file. +6. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`, and `AWS_PROFILE` values to the file. 7. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile. Example: ``` export AWS_PROFILE=versity-test @@ -34,7 +34,7 @@ ### Static Bucket Mode -To preserve buckets while running tests, set `RECREATE_BUCKETS` to `false`. Two utility functions are included, if needed, to create, and delete buckets for this: `tests/setup_static.sh` and `tests/remove_static.sh`. +To preserve buckets while running tests, set `RECREATE_BUCKETS` to `false`. Two utility functions are included, if needed, to create, and delete buckets for this: `tests/setup_static.sh` and `tests/remove_static.sh`. Note that this creates a bucket with object lock enabled, and some tests may fail if the bucket being tested doesn't have object lock enabled. ### S3 Backend diff --git a/tests/commands/create_bucket.sh b/tests/commands/create_bucket.sh index dd26af9f..cdd31efb 100644 --- a/tests/commands/create_bucket.sh +++ b/tests/commands/create_bucket.sh @@ -53,7 +53,6 @@ create_bucket_with_user() { fi if [ $exit_code -ne 0 ]; then log 2 "error creating bucket: $error" - export error return 1 fi return 0 diff --git a/tests/commands/create_multipart_upload.sh b/tests/commands/create_multipart_upload.sh index f1aede4d..594eec8b 100644 --- a/tests/commands/create_multipart_upload.sh +++ b/tests/commands/create_multipart_upload.sh @@ -20,7 +20,6 @@ create_multipart_upload() { return 1 fi upload_id="${upload_id//\"/}" - export upload_id return 0 } @@ -41,7 +40,6 @@ create_multipart_upload_with_user() { return 1 fi upload_id="${upload_id//\"/}" - export upload_id return 0 } @@ -66,10 +64,8 @@ create_multipart_upload_params() { log 2 "error creating multipart upload with params: $multipart_data" return 1 fi - export multipart_data upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq '.UploadId') upload_id="${upload_id//\"/}" - export upload_id return 0 } @@ -91,11 +87,9 @@ create_multipart_upload_custom() { log 2 "error creating custom multipart data command: $multipart_data" return 1 fi - export multipart_data log 5 "multipart data: $multipart_data" upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq '.UploadId') upload_id="${upload_id//\"/}" log 5 "upload id: $upload_id" - export upload_id return 0 } diff --git a/tests/commands/delete_bucket.sh b/tests/commands/delete_bucket.sh index 4d0c974d..5eb0170b 100644 --- a/tests/commands/delete_bucket.sh +++ b/tests/commands/delete_bucket.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -# delete an AWS bucket # param: bucket name -# return 0 for success, 1 for failure +# fail if params are bad, or bucket exists and user is unable to delete bucket delete_bucket() { + log 6 "delete_bucket" record_command "delete-bucket" "client:$1" if [ $# -ne 2 ]; then - log 2 "delete bucket missing command type, bucket name" + log 2 "'delete_bucket' command requires client, bucket" return 1 fi @@ -15,10 +15,9 @@ delete_bucket() { return 1 fi - local exit_code=0 - local error + exit_code=0 if [[ $1 == 's3' ]]; then - error=$(aws --no-verify-ssl s3 rb s3://"$2" 2>&1) || exit_code=$? + error=$(aws --no-verify-ssl s3 rb s3://"$2") || exit_code=$? elif [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then error=$(aws --no-verify-ssl s3api delete-bucket --bucket "$2" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then @@ -32,10 +31,9 @@ delete_bucket() { if [ $exit_code -ne 0 ]; then if [[ "$error" == *"The specified bucket does not exist"* ]]; then return 0 - else - log 2 "error deleting bucket: $error" - return 1 fi + log 2 "error deleting bucket: $error" + return 1 fi return 0 } \ No newline at end of file diff --git a/tests/commands/delete_object.sh b/tests/commands/delete_object.sh index c96fbf98..311b3f15 100644 --- a/tests/commands/delete_object.sh +++ b/tests/commands/delete_object.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash +# params: client, bucket, key delete_object() { + log 6 "delete_object" record_command "delete-object" "client:$1" if [ $# -ne 3 ]; then log 2 "delete object command requires command type, bucket, key" diff --git a/tests/commands/get_bucket_acl.sh b/tests/commands/get_bucket_acl.sh index 224a0a9f..c4edc4c0 100644 --- a/tests/commands/get_bucket_acl.sh +++ b/tests/commands/get_bucket_acl.sh @@ -20,7 +20,6 @@ get_bucket_acl() { return 1 fi acl=$(echo "$acl" | grep -v "InsecureRequestWarning") - export acl } get_bucket_acl_with_user() { @@ -33,6 +32,5 @@ get_bucket_acl_with_user() { log 2 "error getting bucket ACLs: $bucket_acl" return 1 fi - export bucket_acl return 0 } \ No newline at end of file diff --git a/tests/commands/get_bucket_location.sh b/tests/commands/get_bucket_location.sh index 4470fc41..37fcb759 100644 --- a/tests/commands/get_bucket_location.sh +++ b/tests/commands/get_bucket_location.sh @@ -20,7 +20,6 @@ get_bucket_location() { return 1 fi location=$(echo "$location_json" | jq -r '.LocationConstraint') - export location } get_bucket_location_aws() { @@ -35,7 +34,6 @@ get_bucket_location_aws() { return 1 fi bucket_location=$(echo "$location_json" | jq -r '.LocationConstraint') - export bucket_location return 0 } @@ -51,7 +49,6 @@ get_bucket_location_s3cmd() { return 1 fi bucket_location=$(echo "$info" | grep -o 'Location:.*' | awk '{print $2}') - export bucket_location return 0 } @@ -66,7 +63,7 @@ get_bucket_location_mc() { echo "error getting s3cmd info: $info" return 1 fi + # shellcheck disable=SC2034 bucket_location=$(echo "$info" | grep -o 'Location:.*' | awk '{print $2}') - export bucket_location return 0 } \ No newline at end of file diff --git a/tests/commands/get_bucket_ownership_controls.sh b/tests/commands/get_bucket_ownership_controls.sh index b02f51c9..02585792 100644 --- a/tests/commands/get_bucket_ownership_controls.sh +++ b/tests/commands/get_bucket_ownership_controls.sh @@ -14,7 +14,6 @@ get_bucket_ownership_controls() { log 5 "Raw bucket Ownership Controls: $raw_bucket_ownership_controls" bucket_ownership_controls=$(echo "$raw_bucket_ownership_controls" | grep -v "InsecureRequestWarning") - export bucket_ownership_controls return 0 } @@ -32,6 +31,5 @@ get_object_ownership_rule() { return 1 fi log 5 "object ownership rule: $object_ownership_rule" - export object_ownership_rule return 0 } \ No newline at end of file diff --git a/tests/commands/get_bucket_policy.sh b/tests/commands/get_bucket_policy.sh index c3562edf..ffbd92e9 100644 --- a/tests/commands/get_bucket_policy.sh +++ b/tests/commands/get_bucket_policy.sh @@ -21,7 +21,6 @@ get_bucket_policy() { log 2 "error getting policy: $bucket_policy" return 1 fi - export bucket_policy return 0 } @@ -44,7 +43,6 @@ get_bucket_policy_aws() { else bucket_policy=$(echo "$policy_json" | jq -r '.Policy') fi - export bucket_policy return 0 } @@ -65,7 +63,6 @@ get_bucket_policy_with_user() { return 1 fi fi - export bucket_policy return 0 } @@ -109,7 +106,6 @@ get_bucket_policy_s3cmd() { fi done <<< "$info" log 5 "bucket policy: $bucket_policy" - export bucket_policy return 0 } @@ -124,6 +120,5 @@ get_bucket_policy_mc() { echo "error getting policy: $bucket_policy" return 1 fi - export bucket_policy return 0 } \ No newline at end of file diff --git a/tests/commands/get_bucket_tagging.sh b/tests/commands/get_bucket_tagging.sh index 5a24e538..6c3cf61a 100644 --- a/tests/commands/get_bucket_tagging.sh +++ b/tests/commands/get_bucket_tagging.sh @@ -1,22 +1,18 @@ #!/usr/bin/env bash -# get bucket tags -# params: bucket +# params: client, bucket # export 'tags' on success, return 1 for error get_bucket_tagging() { + log 6 "get_bucket_tagging" + assert [ $# -eq 2 ] record_command "get-bucket-tagging" "client:$1" - if [ $# -ne 2 ]; then - echo "get bucket tag command missing command type, bucket name" - return 1 - fi local result if [[ $1 == 'aws' ]]; then tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$? elif [[ $1 == 'mc' ]]; then tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$? else - echo "invalid command type $1" - return 1 + fail "invalid command type $1" fi log 5 "Tags: $tags" tags=$(echo "$tags" | grep -v "InsecureRequestWarning") @@ -29,4 +25,27 @@ get_bucket_tagging() { return 1 fi export tags -} \ No newline at end of file +} + +get_bucket_tagging_with_user() { + log 6 "get_bucket_tagging_with_user" + if [ $# -ne 3 ]; then + log 2 "'get_bucket_tagging_with_user' command requires ID, key, bucket" + return 1 + fi + record_command "get-bucket-tagging" "client:s3api" + local result + if ! tags=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" aws --no-verify-ssl s3api get-bucket-tagging --bucket "$3" 2>&1); then + log 5 "tags error: $tags" + if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then + export tags= + return 0 + fi + fail "unrecognized error getting bucket tagging with user: $tags" + return 1 + fi + log 5 "raw tags data: $tags" + tags=$(echo "$tags" | grep -v "InsecureRequestWarning") + log 5 "modified tags data: $tags" + return 0 +} diff --git a/tests/commands/get_object.sh b/tests/commands/get_object.sh index 22c95303..f63e7c2a 100644 --- a/tests/commands/get_object.sh +++ b/tests/commands/get_object.sh @@ -59,7 +59,6 @@ get_object_with_user() { log 5 "put object exit code: $exit_code" if [ $exit_code -ne 0 ]; then log 2 "error getting object: $get_object_error" - export get_object_error return 1 fi return 0 diff --git a/tests/commands/get_object_attributes.sh b/tests/commands/get_object_attributes.sh index a81b822e..8beb3476 100644 --- a/tests/commands/get_object_attributes.sh +++ b/tests/commands/get_object_attributes.sh @@ -13,6 +13,5 @@ get_object_attributes() { fi attributes=$(echo "$attributes" | grep -v "InsecureRequestWarning") log 5 "$attributes" - export attributes return 0 } \ No newline at end of file diff --git a/tests/commands/get_object_legal_hold.sh b/tests/commands/get_object_legal_hold.sh index bf02c937..e4fd8f5e 100644 --- a/tests/commands/get_object_legal_hold.sh +++ b/tests/commands/get_object_legal_hold.sh @@ -11,6 +11,5 @@ get_object_legal_hold() { log 2 "error getting object legal hold: $legal_hold" return 1 fi - export legal_hold return 0 } \ No newline at end of file diff --git a/tests/commands/get_object_lock_configuration.sh b/tests/commands/get_object_lock_configuration.sh index ed7a2343..72b43c5f 100644 --- a/tests/commands/get_object_lock_configuration.sh +++ b/tests/commands/get_object_lock_configuration.sh @@ -6,11 +6,12 @@ get_object_lock_configuration() { log 2 "'get object lock configuration' command missing bucket name" return 1 fi - lock_config=$(aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1") || local get_result=$? - if [[ $get_result -ne 0 ]]; then + if ! lock_config=$(aws --no-verify-ssl s3api get-object-lock-configuration --bucket "$1" 2>&1); then log 2 "error obtaining lock config: $lock_config" + # shellcheck disable=SC2034 + get_object_lock_config_err=$lock_config return 1 fi - export lock_config + lock_config=$(echo "$lock_config" | grep -v "InsecureRequestWarning") return 0 } \ No newline at end of file diff --git a/tests/commands/get_object_retention.sh b/tests/commands/get_object_retention.sh index c32bea80..7cc27b4e 100644 --- a/tests/commands/get_object_retention.sh +++ b/tests/commands/get_object_retention.sh @@ -12,6 +12,5 @@ get_object_retention() { export get_object_retention_error return 1 fi - export retention return 0 } \ No newline at end of file diff --git a/tests/commands/head_bucket.sh b/tests/commands/head_bucket.sh index 235bb6ec..19531a2b 100644 --- a/tests/commands/head_bucket.sh +++ b/tests/commands/head_bucket.sh @@ -2,12 +2,15 @@ source ./tests/report.sh +# params: client, bucket name +# fail for invalid params, return +# 0 - bucket exists +# 1 - bucket does not exist +# 2 - misc error head_bucket() { + log 6 "head_bucket" record_command "head-bucket" "client:$1" - if [ $# -ne 2 ]; then - echo "head bucket command missing command type, bucket name" - return 1 - fi + assert [ $# -eq 2 ] local exit_code=0 if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]] || [[ $1 == 's3' ]]; then bucket_info=$(aws --no-verify-ssl s3api head-bucket --bucket "$2" 2>&1) || exit_code=$? @@ -16,13 +19,14 @@ head_bucket() { elif [[ $1 == 'mc' ]]; then bucket_info=$(mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? else - echo "invalid command type $1" - return 1 + fail "invalid command type $1" fi if [ $exit_code -ne 0 ]; then - echo "error getting bucket info: $bucket_info" - return 1 + if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then + return 1 + fi + log 2 "error getting bucket info: $bucket_info" + return 2 fi - export bucket_info return 0 } diff --git a/tests/commands/head_object.sh b/tests/commands/head_object.sh index cdabd91a..751ddf2f 100644 --- a/tests/commands/head_object.sh +++ b/tests/commands/head_object.sh @@ -26,6 +26,5 @@ head_object() { return 2 fi fi - export metadata return 0 } \ No newline at end of file diff --git a/tests/commands/list_buckets.sh b/tests/commands/list_buckets.sh index 25678e7d..4acc4d80 100644 --- a/tests/commands/list_buckets.sh +++ b/tests/commands/list_buckets.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash list_buckets() { + log 6 "list_buckets" record_command "list-buckets" "client:$1" if [ $# -ne 1 ]; then echo "list buckets command missing command type" @@ -34,7 +35,6 @@ list_buckets() { bucket_name=$(echo "$line" | awk '{print $NF}') bucket_array+=("${bucket_name%/}") done <<< "$buckets" - export bucket_array return 0 } @@ -72,7 +72,6 @@ list_buckets_with_user() { bucket_name=$(echo "$line" | awk '{print $NF}') bucket_array+=("${bucket_name%/}") done <<< "$buckets" - export bucket_array return 0 } @@ -98,6 +97,5 @@ list_buckets_s3api() { names=$(jq -r '.Buckets[].Name' <<<"$modified_output") IFS=$'\n' read -rd '' -a bucket_array <<<"$names" - export bucket_array return 0 } \ No newline at end of file diff --git a/tests/commands/list_multipart_uploads.sh b/tests/commands/list_multipart_uploads.sh index fa8ac0d6..0dd948ea 100644 --- a/tests/commands/list_multipart_uploads.sh +++ b/tests/commands/list_multipart_uploads.sh @@ -10,7 +10,6 @@ list_multipart_uploads() { log 2 "error listing uploads: $uploads" return 1 fi - export uploads } list_multipart_uploads_with_user() { @@ -21,9 +20,8 @@ list_multipart_uploads_with_user() { fi if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then log 2 "error listing uploads: $uploads" + # shellcheck disable=SC2034 list_multipart_uploads_error=$uploads - export list_multipart_uploads_error return 1 fi - export uploads } \ No newline at end of file diff --git a/tests/commands/list_object_versions.sh b/tests/commands/list_object_versions.sh index d0d8b387..d72fca3a 100644 --- a/tests/commands/list_object_versions.sh +++ b/tests/commands/list_object_versions.sh @@ -11,6 +11,5 @@ list_object_versions() { log 2 "error listing object versions: $versions" return 1 fi - export versions return 0 } \ No newline at end of file diff --git a/tests/commands/list_objects.sh b/tests/commands/list_objects.sh index e0ab3dba..663606cb 100644 --- a/tests/commands/list_objects.sh +++ b/tests/commands/list_objects.sh @@ -1,33 +1,32 @@ #!/usr/bin/env bash +# args: client, bucket name +# return 0 if able to list, 1 if not list_objects() { + log 6 "list_objects" record_command "list-objects" "client:$1" if [ $# -ne 2 ]; then - echo "list objects command requires command type, and bucket or folder" - return 1 - fi - local exit_code=0 - local output - if [[ $1 == "aws" ]] || [[ $1 == 's3' ]]; then - output=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code=$? - elif [[ $1 == 's3api' ]]; then - list_objects_s3api "$2" || exit_code=$? - elif [[ $1 == 's3cmd' ]]; then - output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code=$? - elif [[ $1 == 'mc' ]]; then - output=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? - else - echo "invalid command type $1" - return 1 - fi - if [ $exit_code -ne 0 ]; then - echo "error listing objects: $output" + log 2 "'list_objects' command requires client, bucket" return 1 fi - if [[ $1 == 's3api' ]]; then - return 0 + local output + local result=0 + if [[ $1 == "aws" ]] || [[ $1 == 's3' ]]; then + output=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || result=$? + elif [[ $1 == 's3api' ]]; then + list_objects_s3api "$2" || result=$? + return $result + elif [[ $1 == 's3cmd' ]]; then + output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || result=$? + elif [[ $1 == 'mc' ]]; then + output=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || result=$? + else + fail "invalid command type $1" + return 1 fi + # shellcheck disable=SC2154 + assert_success "error listing objects: $output" object_array=() while IFS= read -r line; do @@ -40,23 +39,21 @@ list_objects() { export object_array } +# args: bucket name +# fail if unable to list list_objects_s3api() { - if [[ $# -ne 1 ]]; then - echo "list objects s3api command requires bucket name" + log 6 "list_objects_s3api" + if [ $# -ne 1 ]; then + log 2 "'list_objects_s3api' requires bucket" return 1 fi - output=$(aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1) || local exit_code=$? - if [[ $exit_code -ne 0 ]]; then - echo "error listing objects: $output" + if ! output=$(aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1); then + log 2 "error listing objects: $output" return 1 fi - modified_output="" - while IFS= read -r line; do - if [[ $line != *InsecureRequestWarning* ]]; then - modified_output+="$line" - fi - done <<< "$output" + log 5 "list_objects_s3api: raw data returned: $output" + modified_output=$(echo "$output" | grep -v "InsecureRequestWarning") object_array=() log 5 "modified output: $modified_output" @@ -66,6 +63,5 @@ list_objects_s3api() { keys=$(echo "$contents" | jq -r '.Key') IFS=$'\n' read -rd '' -a object_array <<<"$keys" fi - - export object_array + return 0 } \ No newline at end of file diff --git a/tests/commands/list_objects_v2.sh b/tests/commands/list_objects_v2.sh index a22b7f52..dd2799f4 100644 --- a/tests/commands/list_objects_v2.sh +++ b/tests/commands/list_objects_v2.sh @@ -14,5 +14,4 @@ list_objects_v2() { echo "error listing objects: $objects" return 1 fi - export objects } \ No newline at end of file diff --git a/tests/commands/list_parts.sh b/tests/commands/list_parts.sh index 7bc47480..059dff0e 100644 --- a/tests/commands/list_parts.sh +++ b/tests/commands/list_parts.sh @@ -10,5 +10,4 @@ list_parts() { log 2 "Error listing multipart upload parts: $listed_parts" return 1 fi - export listed_parts } \ No newline at end of file diff --git a/tests/commands/put_bucket_acl.sh b/tests/commands/put_bucket_acl.sh index 122b7c1d..27f5717c 100644 --- a/tests/commands/put_bucket_acl.sh +++ b/tests/commands/put_bucket_acl.sh @@ -3,13 +3,29 @@ source ./tests/util_file.sh put_bucket_acl_s3api() { - record_command "put-bucket-acl" "client:$1" - if [[ $# -ne 3 ]]; then - log 2 "put bucket acl command requires command type, bucket name, acls or username" + log 6 "put_bucket_acl_s3api" + record_command "put-bucket-acl" "client:s3api" + if [[ $# -ne 2 ]]; then + log 2 "put bucket acl command requires bucket name, acl file" return 1 fi - log 5 "bucket name: $2, acls: $3" - if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$2" --access-control-policy "file://$3" 2>&1); then + log 5 "bucket name: $1, acls: $2" + if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then + log 2 "error putting bucket acl: $error" + return 1 + fi + return 0 +} + +put_bucket_acl_s3api_with_user() { + log 6 "put_bucket_acl_s3api_with_user" + record_command "put-bucket-acl" "client:s3api" + if [[ $# -ne 4 ]]; then + log 2 "put bucket acl command requires bucket name, acl file, username, password" + return 1 + fi + log 5 "bucket name: $1, acls: $2" + if ! error=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --access-control-policy "file://$2" 2>&1); then log 2 "error putting bucket acl: $error" return 1 fi @@ -17,14 +33,15 @@ put_bucket_acl_s3api() { } reset_bucket_acl() { - #if [[ $# -ne 1 ]]; then - # log 2 "'reset_bucket_acl' requires bucket name" - # return 1 - #fi - assert [ $# -eq 1 ] + if [ $# -ne 1 ]; then + log 2 "'reset_bucket_acl' requires bucket name" + return 1 + fi acl_file="acl_file" - run create_test_files "$acl_file" - assert_success "error creating file" + if ! create_test_files "$acl_file"; then + log 2 "error creating test files" + return 1 + fi # shellcheck disable=SC2154 cat < "$test_file_folder/$acl_file" { @@ -42,9 +59,12 @@ reset_bucket_acl() { } } EOF - run put_bucket_acl_s3api "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$acl_file" - assert_success "error putting bucket ACL" + if ! put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$test_file_folder/$acl_file"; then + log 2 "error putting bucket acl (s3api)" + return 1 + fi delete_test_files "$acl_file" + return 0 } put_bucket_canned_acl_s3cmd() { diff --git a/tests/commands/put_bucket_ownership_controls.sh b/tests/commands/put_bucket_ownership_controls.sh index 8b4f0c67..889cca27 100644 --- a/tests/commands/put_bucket_ownership_controls.sh +++ b/tests/commands/put_bucket_ownership_controls.sh @@ -1,15 +1,11 @@ #!/usr/bin/env bash +# fail if unable to put bucket ownership controls put_bucket_ownership_controls() { + log 6 "put_bucket_ownership_controls" record_command "put-bucket-ownership-controls" "client:s3api" - if [[ $# -ne 2 ]]; then - log 2 "'put bucket ownership controls' command requires bucket name, control" - return 1 - fi - if ! controls_error=$(aws --no-verify-ssl s3api put-bucket-ownership-controls --bucket "$1" \ - --ownership-controls="Rules=[{ObjectOwnership=$2}]" 2>&1); then - log 2 "error putting bucket ownership controls: $controls_error" - return 1 - fi - return 0 + assert [ $# -eq 2 ] + run aws --no-verify-ssl s3api put-bucket-ownership-controls --bucket "$1" --ownership-controls="Rules=[{ObjectOwnership=$2}]" + # shellcheck disable=SC2154 + assert_success "error putting bucket ownership controls: $output" } \ No newline at end of file diff --git a/tests/commands/put_bucket_tagging.sh b/tests/commands/put_bucket_tagging.sh index 88a2ff62..1be73f1c 100644 --- a/tests/commands/put_bucket_tagging.sh +++ b/tests/commands/put_bucket_tagging.sh @@ -1,14 +1,15 @@ #!/usr/bin/env bash put_bucket_tagging() { + log 6 "put_bucket_tagging" if [ $# -ne 4 ]; then - echo "bucket tag command missing command type, bucket name, key, value" + log 2 "bucket tag command missing command type, bucket name, key, value" return 1 fi local error - local result + local result=0 record_command "put-bucket-tagging" "client:$1" - if [[ $1 == 'aws' ]]; then + if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$? elif [[ $1 == 'mc' ]]; then error=$(mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$? @@ -17,8 +18,19 @@ put_bucket_tagging() { return 1 fi if [[ $result -ne 0 ]]; then - echo "Error adding bucket tag: $error" + log 2 "Error adding bucket tag: $error" return 1 fi return 0 -} \ No newline at end of file +} + +put_bucket_tagging_with_user() { + log 6 "put_bucket_tagging_with_user" + assert [ $# -eq 5 ] + record_command "put-bucket-tagging" "client:$1" + if ! error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]"); then + log 2 "error putting bucket tagging: $error" + return 1 + fi + return 0 +} diff --git a/tests/commands/put_object.sh b/tests/commands/put_object.sh index ffa34acf..83f49ae7 100644 --- a/tests/commands/put_object.sh +++ b/tests/commands/put_object.sh @@ -3,6 +3,7 @@ source ./tests/report.sh put_object() { + log 6 "put_object" record_command "put-object" "client:$1" if [ $# -ne 4 ]; then log 2 "put object command requires command type, source, destination bucket, destination key" diff --git a/tests/env.sh b/tests/env.sh index 9a5d09b7..c906d4f6 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -59,7 +59,7 @@ check_universal_vars() { if [ "$DIRECT" != "true" ]; then assert [ -n "$AWS_ENDPOINT_URL" ] fi - if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" == "false" ]; then + if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then fail "RUN_VERSITYGW must be 'true' or 'false'" fi @@ -69,6 +69,7 @@ check_universal_vars() { if [ "$RECREATE_BUCKETS" != "true" ] && [ "$RECREATE_BUCKETS" != "false" ]; then fail "RECREATE_BUCKETS must be 'true' or 'false'" fi + assert [ -n "$TEST_FILE_FOLDER" ] # exporting these since they're needed for subshells export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL } diff --git a/tests/report.sh b/tests/report.sh index 3f5a5785..f026c103 100644 --- a/tests/report.sh +++ b/tests/report.sh @@ -42,9 +42,9 @@ record_command() { "client") client=$(echo "$arg" | awk -F: '{print $2}') ;; - #"role") - # role=$(echo "$arg" | awk -F: '{print $2}') - # ;; + *) + log 3 "unrecognized header: $client" + ;; esac done if ! error=$(sqlite3 "$COVERAGE_DB" "INSERT INTO entries (command, client, count) VALUES(\"$1\", \"$client\", 1) ON CONFLICT(command, client) DO UPDATE SET count = count + 1" 2>&1); then diff --git a/tests/run.sh b/tests/run.sh index 803b2ce7..2043c7f4 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -51,7 +51,7 @@ fi case $command_type in s3api|aws) echo "Running aws tests ..." - "$HOME"/bin/bats ./tests/test_aws.sh || exit_code=$? + "$HOME"/bin/bats ./tests/test_s3api.sh || exit_code=$? if [[ $exit_code -eq 0 ]]; then "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$? fi diff --git a/tests/test_aws_root_inner.sh b/tests/test_aws_root_inner.sh index 5ebb94c2..048ed32b 100755 --- a/tests/test_aws_root_inner.sh +++ b/tests/test_aws_root_inner.sh @@ -7,11 +7,12 @@ source ./tests/commands/list_parts.sh test_abort_multipart_upload_aws_root() { local bucket_file="bucket-file" - create_test_files "$bucket_file" || fail "error creating test files" + create_test_files "$bucket_file" # shellcheck disable=SC2154 - dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file" + run dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 + assert_success "error creating file" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "Failed to create bucket '$BUCKET_ONE_NAME'" + setup_bucket "aws" "$BUCKET_ONE_NAME" run_then_abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "abort failed" @@ -29,7 +30,7 @@ test_complete_multipart_upload_aws_root() { create_test_files "$bucket_file" || fail "error creating test files" dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "failed to create bucket '$BUCKET_ONE_NAME'" + setup_bucket "aws" "$BUCKET_ONE_NAME" multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "error performing multipart upload" @@ -124,7 +125,7 @@ test_delete_objects_aws_root() { local object_two="test-file-two" create_test_files "$object_one" "$object_two" || fail "error creating test files" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error creating bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" put_object "s3api" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME" "$object_one" || fail "error adding object one" put_object "s3api" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME" "$object_two" || fail "error adding object two" @@ -145,7 +146,7 @@ test_get_bucket_acl_aws_root() { if [[ $RECREATE_BUCKETS == "false" ]]; then skip fi - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error creating bucket" + setup_bucket "aws" "$BUCKET_ONE_NAME" get_bucket_acl "s3api" "$BUCKET_ONE_NAME" || fail "error retreving ACL" @@ -163,8 +164,8 @@ test_get_object_full_range_aws_root() { create_test_files "$bucket_file" || local created=$? [[ $created -eq 0 ]] || fail "Error creating test files" echo -n "0123456789" > "$test_file_folder/$bucket_file" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" + put_object "s3api" "$test_file_folder/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error putting object" get_object_with_range "$BUCKET_ONE_NAME" "$bucket_file" "bytes=9-15" "$test_file_folder/$bucket_file-range" || fail "error getting range" [[ "$(cat "$test_file_folder/$bucket_file-range")" == "9" ]] || fail "byte range not copied properly" @@ -175,8 +176,8 @@ test_get_object_invalid_range_aws_root() { create_test_files "$bucket_file" || local created=$? [[ $created -eq 0 ]] || fail "Error creating test files" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + + setup_bucket "s3api" "$BUCKET_ONE_NAME" put_object "s3api" "$test_file_folder/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error putting object" get_object_with_range "$BUCKET_ONE_NAME" "$bucket_file" "bytes=0-0" "$test_file_folder/$bucket_file-range" || local get_result=$? [[ $get_result -ne 0 ]] || fail "Get object with zero range returned no error" @@ -187,10 +188,9 @@ test_put_object_aws_root() { create_test_files "$bucket_file" || local created=$? [[ $created -eq 0 ]] || fail "Error creating test files" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - setup_bucket "s3api" "$BUCKET_TWO_NAME" || local setup_result_two=$? - [[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error" + + setup_bucket "s3api" "$BUCKET_ONE_NAME" + setup_bucket "s3api" "$BUCKET_TWO_NAME" put_object "s3api" "$test_file_folder/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || local copy_result=$? [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" copy_error=$(aws --no-verify-ssl s3api copy-object --copy-source "$BUCKET_ONE_NAME/$bucket_file" --key "$bucket_file" --bucket "$BUCKET_TWO_NAME" 2>&1) || local copy_result=$? @@ -221,7 +221,7 @@ test_get_object_attributes_aws_root() { bucket_file="bucket_file" create_test_files "$bucket_file" || fail "error creating test files" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" put_object "s3api" "$test_file_folder/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || fail "failed to add object to bucket" get_object_attributes "$BUCKET_ONE_NAME" "$bucket_file" || failed "failed to get object attributes" # shellcheck disable=SC2154 @@ -404,7 +404,7 @@ test_s3api_list_objects_v1_aws_root() { create_test_files "$object_one" "$object_two" || local created=$? [[ $created -eq 0 ]] || fail "Error creating test files" printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? + setup_bucket "aws" "$BUCKET_ONE_NAME" [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" put_object "s3api" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME" "$object_one" || local copy_result_one=$? [[ $copy_result_one -eq 0 ]] || fail "Failed to add object $object_one" @@ -434,7 +434,7 @@ test_s3api_list_objects_v2_aws_root() { create_test_files "$object_one" "$object_two" || local created=$? [[ $created -eq 0 ]] || fail "Error creating test files" printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? + setup_bucket "aws" "$BUCKET_ONE_NAME" [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" put_object "s3api" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME" "$object_one" || local copy_object_one=$? [[ $copy_object_one -eq 0 ]] || fail "Failed to add object $object_one" @@ -460,7 +460,7 @@ test_multipart_upload_list_parts_aws_root() { create_test_files "$bucket_file" || fail "error creating test file" dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "failed to create bucket '$BUCKET_ONE_NAME'" + setup_bucket "aws" "$BUCKET_ONE_NAME" start_multipart_upload_and_list_parts "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "listing multipart upload parts failed" diff --git a/tests/test_common.sh b/tests/test_common.sh index b7813bc8..264b44dd 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -29,7 +29,7 @@ test_common_multipart_upload() { create_large_file "$bucket_file" || local created=$? [[ $created -eq 0 ]] || fail "Error creating test file for multipart upload" - setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? + setup_bucket "$1" "$BUCKET_ONE_NAME" [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" put_object "$1" "$test_file_folder/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || local put_result=$? @@ -47,11 +47,9 @@ test_common_create_delete_bucket() { return fi - if [[ $# -ne 1 ]]; then - fail "create/delete bucket test requires command type" - fi + assert [ $# -eq 1 ] - setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "failed to create bucket" + setup_bucket "$1" "$BUCKET_ONE_NAME" bucket_exists "$1" "$BUCKET_ONE_NAME" || fail "failed bucket existence check" @@ -66,8 +64,8 @@ test_common_copy_object() { create_test_files "$object_name" || fail "error creating test file" echo "test data" > "$test_file_folder/$object_name" - setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error setting up bucket one" - setup_bucket "$1" "$BUCKET_TWO_NAME" || fail "error setting up bucket two" + setup_bucket "$1" "$BUCKET_ONE_NAME" + setup_bucket "$1" "$BUCKET_TWO_NAME" if [[ $1 == 's3' ]]; then copy_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || fail "failed to copy object to bucket one" @@ -116,8 +114,7 @@ test_common_put_object() { fail "put object test requires command type, file" fi - setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "$1" "$BUCKET_ONE_NAME" put_object "$1" "$test_file_folder/$2" "$BUCKET_ONE_NAME" "$2" || local copy_result=$? [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" @@ -143,7 +140,7 @@ test_common_put_get_object() { create_test_files "$object_name" || fail "error creating test file" echo "test data" > "$test_file_folder"/"$object_name" - setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "$1" "$BUCKET_ONE_NAME" if [[ $1 == 's3' ]]; then copy_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || fail "failed to add object to bucket" @@ -164,8 +161,7 @@ test_common_get_set_versioning() { create_test_files "$object_name" || local create_result=$? [[ $create_result -eq 0 ]] || fail "Error creating test file" - setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "$1" "$BUCKET_ONE_NAME" get_bucket_versioning "$1" "$BUCKET_ONE_NAME" || local get_result=$? [[ $get_result -eq 0 ]] || fail "error getting bucket versioning" @@ -187,10 +183,8 @@ test_common_list_buckets() { fail "List buckets test requires one argument" fi - setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result_one=$? - [[ $setup_result_one -eq 0 ]] || fail "Bucket one setup error" - setup_bucket "$1" "$BUCKET_TWO_NAME" || local setup_result_two=$? - [[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error" + setup_bucket "$1" "$BUCKET_ONE_NAME" + setup_bucket "$1" "$BUCKET_TWO_NAME" list_buckets "$1" local bucket_one_found=false @@ -230,8 +224,7 @@ test_common_list_objects() { create_test_files $object_one $object_two echo "test data" > "$test_file_folder"/"$object_one" echo "test data 2" > "$test_file_folder"/"$object_two" - setup_bucket "$1" "$BUCKET_ONE_NAME" || local result_one=$? - [[ result_one -eq 0 ]] || fail "Error creating bucket" + setup_bucket "$1" "$BUCKET_ONE_NAME" put_object "$1" "$test_file_folder"/$object_one "$BUCKET_ONE_NAME" "$object_one" || local result_two=$? [[ result_two -eq 0 ]] || fail "Error adding object one" put_object "$1" "$test_file_folder"/$object_two "$BUCKET_ONE_NAME" "$object_two" || local result_three=$? @@ -265,7 +258,7 @@ test_common_set_get_delete_bucket_tags() { local key="test_key" local value="test_value" - setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "Failed to create bucket '$BUCKET_ONE_NAME'" + setup_bucket "$1" "$BUCKET_ONE_NAME" get_bucket_tagging "$1" "$BUCKET_ONE_NAME" || fail "Error getting bucket tags first time" @@ -306,7 +299,7 @@ test_common_set_get_object_tags() { local value="test_value" create_test_files "$bucket_file" || fail "error creating test files" - setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "Failed to create bucket '$BUCKET_ONE_NAME'" + setup_bucket "$1" "$BUCKET_ONE_NAME" put_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'" get_object_tagging "$1" "$BUCKET_ONE_NAME" $bucket_file || fail "Error getting object tags" @@ -345,7 +338,7 @@ test_common_presigned_url_utf8_chars() { create_test_files "$bucket_file" || local created=$? dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file" - setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? + setup_bucket "$1" "$BUCKET_ONE_NAME" [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" put_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || put_result=$? @@ -376,7 +369,7 @@ test_common_list_objects_file_count() { fi create_test_file_count 1001 || local create_result=$? [[ $create_result -eq 0 ]] || fail "error creating test files" - setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? + setup_bucket "$1" "$BUCKET_ONE_NAME" [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" put_object_multiple "$1" "$test_file_folder/file_*" "$BUCKET_ONE_NAME" || local put_result=$? [[ $put_result -eq 0 ]] || fail "Failed to copy files to bucket" @@ -398,7 +391,7 @@ test_common_delete_object_tagging() { create_test_files "$bucket_file" || fail "Error creating test files" - setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "$1" "$BUCKET_ONE_NAME" put_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || fail "Failed to add object to bucket" @@ -416,8 +409,7 @@ test_common_delete_object_tagging() { test_common_get_bucket_location() { [[ $# -eq 1 ]] || fail "test common get bucket location missing command type" - setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "$1" "$BUCKET_ONE_NAME" get_bucket_location "$1" "$BUCKET_ONE_NAME" # shellcheck disable=SC2154 [[ $bucket_location == "null" ]] || [[ $bucket_location == "us-east-1" ]] || fail "wrong location: '$bucket_location'" @@ -428,7 +420,7 @@ test_put_bucket_acl_s3cmd() { # https://github.com/versity/versitygw/issues/695 skip fi - setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || fail "error creating bucket" + setup_bucket "s3cmd" "$BUCKET_ONE_NAME" put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" || fail "error putting bucket ownership controls" username=$USERNAME_ONE @@ -477,7 +469,7 @@ test_common_put_bucket_acl() { skip fi [[ $# -eq 1 ]] || fail "test common put bucket acl missing command type" - setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error creating bucket" + setup_bucket "$1" "$BUCKET_ONE_NAME" put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" || fail "error putting bucket ownership controls" username=$USERNAME_ONE @@ -517,7 +509,7 @@ cat < "$test_file_folder"/"$acl_file" EOF log 6 "before 1st put acl" - put_bucket_acl_s3api "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || fail "error putting first acl" + put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || fail "error putting first acl" get_bucket_acl "$1" "$BUCKET_ONE_NAME" || fail "error retrieving second ACL" log 5 "Acls after 1st put: $acl" @@ -542,7 +534,7 @@ cat < "$test_file_folder"/"$acl_file" } EOF - put_bucket_acl_s3api "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || fail "error putting second acl" + put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || fail "error putting second acl" get_bucket_acl "$1" "$BUCKET_ONE_NAME" || fail "error retrieving second ACL" log 5 "Acls after 2nd put: $acl" @@ -587,7 +579,7 @@ test_common_get_put_delete_bucket_policy() { EOF log 5 "POLICY: $(cat "$test_file_folder/$policy_file")" - setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "$1" "$BUCKET_ONE_NAME" check_for_empty_policy "$1" "$BUCKET_ONE_NAME" || fail "policy not empty" diff --git a/tests/test_mc.sh b/tests/test_mc.sh index b5863fec..965bf8f4 100755 --- a/tests/test_mc.sh +++ b/tests/test_mc.sh @@ -29,7 +29,7 @@ export RUN_MC=true if [[ $RECREATE_BUCKETS == "false" ]]; then skip "will not test bucket deletion in static bucket test config" fi - setup_bucket "mc" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "mc" "$BUCKET_ONE_NAME" delete_bucket "mc" "$BUCKET_ONE_NAME" || fail "error deleting bucket" } @@ -108,16 +108,14 @@ export RUN_MC=true } @test "test_get_bucket_info_mc" { - setup_bucket "mc" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "mc" "$BUCKET_ONE_NAME" head_bucket "mc" "$BUCKET_ONE_NAME" [[ $bucket_info == *"$BUCKET_ONE_NAME"* ]] || fail "failure to retrieve correct bucket info: $bucket_info" delete_bucket_or_contents "mc" "$BUCKET_ONE_NAME" } @test "test_get_bucket_info_doesnt_exist_mc" { - setup_bucket "mc" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "mc" "$BUCKET_ONE_NAME" head_bucket "mc" "$BUCKET_ONE_NAME"a || local info_result=$? [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned" [[ $bucket_info == *"does not exist"* ]] || fail "404 not returned for non-existent bucket info" diff --git a/tests/test_s3.sh b/tests/test_s3.sh index c5d51c13..a5440646 100755 --- a/tests/test_s3.sh +++ b/tests/test_s3.sh @@ -44,6 +44,6 @@ source ./tests/test_common.sh if [[ $RECREATE_BUCKETS == "false" ]]; then skip "will not test bucket deletion in static bucket test config" fi - setup_bucket "s3" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3" "$BUCKET_ONE_NAME" delete_bucket "s3" "$BUCKET_ONE_NAME" || fail "error deleting bucket" } diff --git a/tests/test_s3api.sh b/tests/test_s3api.sh new file mode 100755 index 00000000..b89974ba --- /dev/null +++ b/tests/test_s3api.sh @@ -0,0 +1,458 @@ +#!/usr/bin/env bats + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/setup.sh +source ./tests/util.sh +source ./tests/util_aws.sh +source ./tests/util_bucket_create.sh +source ./tests/util_file.sh +source ./tests/util_lock_config.sh +source ./tests/util_users.sh +source ./tests/test_aws_root_inner.sh +source ./tests/test_common.sh +source ./tests/test_s3api_policy.sh +source ./tests/commands/copy_object.sh +source ./tests/commands/delete_bucket_policy.sh +source ./tests/commands/delete_object_tagging.sh +source ./tests/commands/get_bucket_acl.sh +source ./tests/commands/get_bucket_policy.sh +source ./tests/commands/get_bucket_versioning.sh +source ./tests/commands/get_object.sh +source ./tests/commands/get_object_attributes.sh +source ./tests/commands/get_object_legal_hold.sh +source ./tests/commands/get_object_lock_configuration.sh +source ./tests/commands/get_object_retention.sh +source ./tests/commands/get_object_tagging.sh +source ./tests/commands/list_multipart_uploads.sh +source ./tests/commands/list_object_versions.sh +source ./tests/commands/put_bucket_acl.sh +source ./tests/commands/put_bucket_policy.sh +source ./tests/commands/put_bucket_versioning.sh +source ./tests/commands/put_object.sh +source ./tests/commands/put_object_legal_hold.sh +source ./tests/commands/put_object_lock_configuration.sh +source ./tests/commands/put_object_retention.sh +source ./tests/commands/put_public_access_block.sh +source ./tests/commands/select_object_content.sh + +export RUN_USERS=true + +# abort-multipart-upload +@test "test_abort_multipart_upload" { + test_abort_multipart_upload_aws_root +} + +# complete-multipart-upload +@test "test_complete_multipart_upload" { + test_complete_multipart_upload_aws_root +} + +# copy-object +@test "test_copy_object" { + test_common_copy_object "s3api" +} + +@test "test_copy_object_empty" { + copy_object_empty || fail "copy objects with no parameters test failure" +} + +# create-bucket +@test "test_create_delete_bucket_aws" { + test_common_create_delete_bucket "aws" +} + +@test "test_create_bucket_invalid_name" { + test_create_bucket_invalid_name_aws_root +} + +# create-multipart-upload +@test "test_create_multipart_upload_properties" { + test_create_multipart_upload_properties_aws_root +} + +# delete-bucket - test_create_delete_bucket_aws + +# delete-bucket-policy +@test "test_get_put_delete_bucket_policy" { + test_common_get_put_delete_bucket_policy "aws" +} + +# delete-bucket-tagging +@test "test-set-get-delete-bucket-tags" { + test_common_set_get_delete_bucket_tags "aws" +} + +# delete-object - tested with bucket cleanup before or after tests + +# delete-object-tagging +@test "test_delete_object_tagging" { + test_common_delete_object_tagging "aws" +} + +# delete-objects +@test "test_delete_objects" { + test_delete_objects_aws_root +} + +# get-bucket-acl +@test "test_get_bucket_acl" { + test_get_bucket_acl_aws_root +} + +# get-bucket-location +@test "test_get_bucket_location" { + test_common_get_bucket_location "aws" +} + +# get-bucket-policy - test_get_put_delete_bucket_policy + +# get-bucket-tagging - test_set_get_delete_bucket_tags + +# get-object +@test "test_get_object_full_range" { + test_get_object_full_range_aws_root +} + +@test "test_get_object_invalid_range" { + test_get_object_invalid_range_aws_root +} + +# get-object-attributes +@test "test_get_object_attributes" { + test_get_object_attributes_aws_root +} + +@test "test_head_bucket_invalid_name" { + if head_bucket "aws" ""; then + fail "able to get bucket info for invalid name" + fi +} + +@test "test_put_object" { + test_put_object_aws_root +} + +# test adding and removing an object on versitygw +@test "test_put_object_with_data" { + test_common_put_object_with_data "aws" +} + +@test "test_put_object_no_data" { + test_common_put_object_no_data "aws" +} + +# test listing buckets on versitygw +@test "test_list_buckets" { + test_common_list_buckets "s3api" +} + +# test listing a bucket's objects on versitygw +@test "test_list_objects" { + test_common_list_objects "aws" +} + +@test "test_get_put_object_legal_hold" { + test_get_put_object_legal_hold_aws_root +} + +@test "test_get_put_object_retention" { + test_get_put_object_retention_aws_root +} + +@test "test_put_bucket_acl" { + test_common_put_bucket_acl "s3api" +} + +# test v1 s3api list objects command +@test "test-s3api-list-objects-v1" { + test_s3api_list_objects_v1_aws_root +} + +# test v2 s3api list objects command +@test "test-s3api-list-objects-v2" { + test_s3api_list_objects_v2_aws_root +} + +# test abilty to set and retrieve object tags +@test "test-set-get-object-tags" { + test_common_set_get_object_tags "aws" +} + +# test multi-part upload list parts command +@test "test-multipart-upload-list-parts" { + test_multipart_upload_list_parts_aws_root +} + +# test listing of active uploads +@test "test-multipart-upload-list-uploads" { + local bucket_file_one="bucket-file-one" + local bucket_file_two="bucket-file-two" + + if [[ $RECREATE_BUCKETS == false ]]; then + abort_all_multipart_uploads "$BUCKET_ONE_NAME" || fail "error aborting all uploads" + fi + + create_test_files "$bucket_file_one" "$bucket_file_two" || fail "error creating test files" + setup_bucket "aws" "$BUCKET_ONE_NAME" + + create_and_list_multipart_uploads "$BUCKET_ONE_NAME" "$test_file_folder"/"$bucket_file_one" "$test_file_folder"/"$bucket_file_two" || fail "failed to list multipart uploads" + + local key_one + local key_two + # shellcheck disable=SC2154 + log 5 "Uploads: $uploads" + raw_uploads=$(echo "$uploads" | grep -v "InsecureRequestWarning") + key_one=$(echo "$raw_uploads" | jq -r '.Uploads[0].Key' 2>&1) || fail "error getting key one: $key_one" + key_two=$(echo "$raw_uploads" | jq -r '.Uploads[1].Key' 2>&1) || fail "error getting key two: $key_two" + key_one=${key_one//\"/} + key_two=${key_two//\"/} + [[ "$test_file_folder/$bucket_file_one" == *"$key_one" ]] || fail "Key mismatch ($test_file_folder/$bucket_file_one, $key_one)" + [[ "$test_file_folder/$bucket_file_two" == *"$key_two" ]] || fail "Key mismatch ($test_file_folder/$bucket_file_two, $key_two)" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files "$bucket_file_one" "$bucket_file_two" +} + +@test "test-multipart-upload-from-bucket" { + local bucket_file="bucket-file" + + create_test_files "$bucket_file" || fail "error creating test files" + dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error adding data to test file" + setup_bucket "aws" "$BUCKET_ONE_NAME" + + multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "error performing multipart upload" + + get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$test_file_folder/$bucket_file-copy" || fail "error getting object" + compare_files "$test_file_folder"/$bucket_file-copy "$test_file_folder"/$bucket_file || fail "data doesn't match" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files $bucket_file +} + +@test "test_multipart_upload_from_bucket_range_too_large" { + local bucket_file="bucket-file" + + create_large_file "$bucket_file" + setup_bucket "aws" "$BUCKET_ONE_NAME" + + multipart_upload_from_bucket_range "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 "bytes=0-1000000000" || local upload_result=$? + [[ $upload_result -eq 1 ]] || fail "multipart upload with overly large range should have failed" + log 5 "error: $upload_part_copy_error" + [[ $upload_part_copy_error == *"Range specified is not valid"* ]] || [[ $upload_part_copy_error == *"InvalidRange"* ]] || fail "unexpected error: $upload_part_copy_error" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files $bucket_file +} + +@test "test_multipart_upload_from_bucket_range_valid" { + local bucket_file="bucket-file" + + create_large_file "$bucket_file" + setup_bucket "aws" "$BUCKET_ONE_NAME" + + range_max=$((5*1024*1024-1)) + multipart_upload_from_bucket_range "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 "bytes=0-$range_max" || fail "upload failure" + + get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$test_file_folder/$bucket_file-copy" || fail "error retrieving object after upload" + if [[ $(uname) == 'Darwin' ]]; then + object_size=$(stat -f%z "$test_file_folder/$bucket_file-copy") + else + object_size=$(stat --format=%s "$test_file_folder/$bucket_file-copy") + fi + [[ object_size -eq $((range_max*4+4)) ]] || fail "object size mismatch ($object_size, $((range_max*4+4)))" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files $bucket_file +} + +@test "test-presigned-url-utf8-chars" { + test_common_presigned_url_utf8_chars "aws" +} + +@test "test-list-objects-delimiter" { + folder_name="two" + object_name="three" + create_test_folder "$folder_name" + create_test_files "$folder_name"/"$object_name" + + setup_bucket "aws" "$BUCKET_ONE_NAME" + + put_object "aws" "$test_file_folder/$folder_name/$object_name" "$BUCKET_ONE_NAME" "$folder_name/$object_name" || fail "failed to add object to bucket" + + list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/" + prefix=$(echo "${objects[@]}" | jq -r ".CommonPrefixes[0].Prefix" 2>&1) || fail "error getting object prefix from object list: $prefix" + [[ $prefix == "$folder_name/" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)" + + list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#" + key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1) || fail "error getting key from object list: $key" + [[ $key == "$folder_name/$object_name" ]] || fail "key doesn't match (expected $key, actual $folder_name/$object_name)" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files $folder_name +} + +@test "test_put_policy_invalid_action" { + test_s3api_policy_invalid_action +} + +@test "test_policy_get_object_with_user" { + test_s3api_policy_get_object_with_user +} + +@test "test_policy_get_object_specific_file" { + test_s3api_policy_get_object_specific_file +} + +@test "test_policy_get_object_file_wildcard" { + test_s3api_policy_get_object_file_wildcard +} + +@test "test_policy_get_object_folder_wildcard" { + test_s3api_policy_get_object_folder_wildcard +} + +@test "test_policy_allow_deny" { + test_s3api_policy_allow_deny +} + +@test "test_policy_deny" { + test_s3api_policy_deny +} + +@test "test_policy_put_wildcard" { + test_s3api_policy_put_wildcard +} + +@test "test_policy_delete" { + test_s3api_policy_delete +} + +@test "test_policy_get_bucket_policy" { + test_s3api_policy_get_bucket_policy +} + +@test "test_policy_list_multipart_uploads" { + test_s3api_policy_list_multipart_uploads +} + +@test "test_policy_put_bucket_policy" { + test_s3api_policy_put_bucket_policy +} + +@test "test_policy_delete_bucket_policy" { + test_s3api_policy_delete_bucket_policy +} + +@test "test_policy_get_bucket_acl" { + test_s3api_policy_get_bucket_acl +} + +# ensure that lists of files greater than a size of 1000 (pagination) are returned properly +#@test "test_list_objects_file_count" { +# test_common_list_objects_file_count "aws" +#} + +# ensure that lists of files greater than a size of 1000 (pagination) are returned properly +#@test "test_list_objects_file_count" { +# test_common_list_objects_file_count "aws" +#} + +#@test "test_filename_length" { +# file_name=$(printf "%0.sa" $(seq 1 1025)) +# echo "$file_name" + +# create_test_files "$file_name" || created=$? +# [[ $created -eq 0 ]] || fail "error creating file" + +# setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? +# [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + +# put_object "aws" "$test_file_folder"/"$file_name" "$BUCKET_ONE_NAME"/"$file_name" || local put_object=$? +# [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" +#} + +@test "test_head_bucket" { + setup_bucket "aws" "$BUCKET_ONE_NAME" + head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info" + log 5 "INFO: $bucket_info" + region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region" + [[ $region != "" ]] || fail "empty bucket region" + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" +} + +@test "test_retention_bypass" { + test_retention_bypass_aws_root +} + +@test "test_head_bucket_doesnt_exist" { + setup_bucket "aws" "$BUCKET_ONE_NAME" + head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$? + [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned" + [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info" + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" +} + +@test "test_add_object_metadata" { + object_one="object-one" + test_key="x-test-data" + test_value="test-value" + + create_test_files "$object_one" || fail "error creating test files" + + setup_bucket "aws" "$BUCKET_ONE_NAME" + + object="$test_file_folder"/"$object_one" + put_object_with_metadata "aws" "$object" "$BUCKET_ONE_NAME" "$object_one" "$test_key" "$test_value" || fail "failed to add object to bucket" + object_exists "aws" "$BUCKET_ONE_NAME" "$object_one" || fail "object not found after being added to bucket" + + get_object_metadata "aws" "$BUCKET_ONE_NAME" "$object_one" || fail "error getting object metadata" + key=$(echo "$metadata" | jq -r 'keys[]' 2>&1) || fail "error getting key from metadata: $key" + value=$(echo "$metadata" | jq -r '.[]' 2>&1) || fail "error getting value from metadata: $value" + [[ $key == "$test_key" ]] || fail "keys doesn't match (expected $key, actual \"$test_key\")" + [[ $value == "$test_value" ]] || fail "values doesn't match (expected $value, actual \"$test_value\")" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files "$object_one" +} + +@test "test_policy_abort_multipart_upload" { + test_s3api_policy_abort_multipart_upload +} + +@test "test_policy_two_principals" { + test_s3api_policy_two_principals +} + +@test "test_policy_put_bucket_tagging" { + test_s3api_policy_put_bucket_tagging +} + +@test "test_policy_get_bucket_tagging" { + test_s3api_policy_get_bucket_tagging +} + +@test "test_policy_put_acl" { + test_s3api_policy_put_acl +} + +@test "test_put_object_lock_configuration" { + bucket_name=$BUCKET_ONE_NAME + if [[ $RECREATE_BUCKETS == "true" ]]; then + delete_bucket "s3api" "$bucket_name" || fail "error deleting bucket" + create_bucket_object_lock_enabled "$bucket_name" || fail "error setting up bucket" + fi + local enabled="Enabled" + local governance="GOVERNANCE" + local days="1" + + run put_object_lock_configuration "$bucket_name" "$enabled" "$governance" "$days" + assert_success "error putting object lock config" + + run get_and_check_object_lock_config "$bucket_name" "$enabled" "$governance" "$days" + assert_success "error getting and checking object lock config" + + delete_bucket_or_contents "aws" "$bucket_name" +} + diff --git a/tests/test_aws.sh b/tests/test_s3api_policy.sh old mode 100755 new mode 100644 similarity index 54% rename from tests/test_aws.sh rename to tests/test_s3api_policy.sh index 1d6daa4f..801ea57e --- a/tests/test_aws.sh +++ b/tests/test_s3api_policy.sh @@ -1,323 +1,10 @@ #!/usr/bin/env bats -load ./bats-support/load -load ./bats-assert/load +source ./tests/util_tags.sh +source ./tests/commands/get_bucket_tagging.sh +source ./tests/commands/put_bucket_tagging.sh -source ./tests/setup.sh -source ./tests/util.sh -source ./tests/util_aws.sh -source ./tests/util_bucket_create.sh -source ./tests/util_file.sh -source ./tests/util_users.sh -source ./tests/test_aws_root_inner.sh -source ./tests/test_common.sh -source ./tests/commands/copy_object.sh -source ./tests/commands/delete_bucket_policy.sh -source ./tests/commands/delete_object_tagging.sh -source ./tests/commands/get_bucket_acl.sh -source ./tests/commands/get_bucket_policy.sh -source ./tests/commands/get_bucket_versioning.sh -source ./tests/commands/get_object.sh -source ./tests/commands/get_object_attributes.sh -source ./tests/commands/get_object_legal_hold.sh -source ./tests/commands/get_object_lock_configuration.sh -source ./tests/commands/get_object_retention.sh -source ./tests/commands/get_object_tagging.sh -source ./tests/commands/list_multipart_uploads.sh -source ./tests/commands/list_object_versions.sh -source ./tests/commands/put_bucket_acl.sh -source ./tests/commands/put_bucket_policy.sh -source ./tests/commands/put_bucket_versioning.sh -source ./tests/commands/put_object.sh -source ./tests/commands/put_object_legal_hold.sh -source ./tests/commands/put_object_lock_configuration.sh -source ./tests/commands/put_object_retention.sh -source ./tests/commands/put_public_access_block.sh -source ./tests/commands/select_object_content.sh - -export RUN_USERS=true - -# abort-multipart-upload -@test "test_abort_multipart_upload" { - test_abort_multipart_upload_aws_root -} - -# complete-multipart-upload -@test "test_complete_multipart_upload" { - test_complete_multipart_upload_aws_root -} - -# copy-object -@test "test_copy_object" { - test_common_copy_object "s3api" -} - -@test "test_copy_object_empty" { - copy_object_empty || fail "copy objects with no parameters test failure" -} - -# create-bucket -@test "test_create_delete_bucket_aws" { - test_common_create_delete_bucket "aws" -} - -@test "test_create_bucket_invalid_name" { - test_create_bucket_invalid_name_aws_root -} - -# create-multipart-upload -@test "test_create_multipart_upload_properties" { - test_create_multipart_upload_properties_aws_root -} - -# delete-bucket - test_create_delete_bucket_aws - -# delete-bucket-policy -@test "test_get_put_delete_bucket_policy" { - test_common_get_put_delete_bucket_policy "aws" -} - -# delete-bucket-tagging -@test "test-set-get-delete-bucket-tags" { - test_common_set_get_delete_bucket_tags "aws" -} - -# delete-object - tested with bucket cleanup before or after tests - -# delete-object-tagging -@test "test_delete_object_tagging" { - test_common_delete_object_tagging "aws" -} - -# delete-objects -@test "test_delete_objects" { - test_delete_objects_aws_root -} - -# get-bucket-acl -@test "test_get_bucket_acl" { - test_get_bucket_acl_aws_root -} - -# get-bucket-location -@test "test_get_bucket_location" { - test_common_get_bucket_location "aws" -} - -# get-bucket-policy - test_get_put_delete_bucket_policy - -# get-bucket-tagging - test_set_get_delete_bucket_tags - -# get-object -@test "test_get_object_full_range" { - test_get_object_full_range_aws_root -} - -@test "test_get_object_invalid_range" { - test_get_object_invalid_range_aws_root -} - -# get-object-attributes -@test "test_get_object_attributes" { - test_get_object_attributes_aws_root -} - -@test "test_put_object" { - test_put_object_aws_root -} - -# test adding and removing an object on versitygw -@test "test_put_object_with_data" { - test_common_put_object_with_data "aws" -} - -@test "test_put_object_no_data" { - test_common_put_object_no_data "aws" -} - -# test listing buckets on versitygw -@test "test_list_buckets" { - test_common_list_buckets "s3api" -} - -# test listing a bucket's objects on versitygw -@test "test_list_objects" { - test_common_list_objects "aws" -} - -@test "test_get_put_object_legal_hold" { - test_get_put_object_legal_hold_aws_root -} - -@test "test_get_put_object_retention" { - test_get_put_object_retention_aws_root -} - -@test "test_put_bucket_acl" { - test_common_put_bucket_acl "s3api" -} - -# test v1 s3api list objects command -@test "test-s3api-list-objects-v1" { - test_s3api_list_objects_v1_aws_root -} - -# test v2 s3api list objects command -@test "test-s3api-list-objects-v2" { - test_s3api_list_objects_v2_aws_root -} - -# test abilty to set and retrieve object tags -@test "test-set-get-object-tags" { - test_common_set_get_object_tags "aws" -} - -# test multi-part upload list parts command -@test "test-multipart-upload-list-parts" { - test_multipart_upload_list_parts_aws_root -} - -# test listing of active uploads -@test "test-multipart-upload-list-uploads" { - local bucket_file_one="bucket-file-one" - local bucket_file_two="bucket-file-two" - - if [[ $RECREATE_BUCKETS == false ]]; then - abort_all_multipart_uploads "$BUCKET_ONE_NAME" || fail "error aborting all uploads" - fi - - create_test_files "$bucket_file_one" "$bucket_file_two" || fail "error creating test files" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "failed to create bucket '$BUCKET_ONE_NAME'" - - create_and_list_multipart_uploads "$BUCKET_ONE_NAME" "$test_file_folder"/"$bucket_file_one" "$test_file_folder"/"$bucket_file_two" || fail "failed to list multipart uploads" - - local key_one - local key_two - # shellcheck disable=SC2154 - log 5 "Uploads: $uploads" - raw_uploads=$(echo "$uploads" | grep -v "InsecureRequestWarning") - key_one=$(echo "$raw_uploads" | jq -r '.Uploads[0].Key' 2>&1) || fail "error getting key one: $key_one" - key_two=$(echo "$raw_uploads" | jq -r '.Uploads[1].Key' 2>&1) || fail "error getting key two: $key_two" - key_one=${key_one//\"/} - key_two=${key_two//\"/} - [[ "$test_file_folder/$bucket_file_one" == *"$key_one" ]] || fail "Key mismatch ($test_file_folder/$bucket_file_one, $key_one)" - [[ "$test_file_folder/$bucket_file_two" == *"$key_two" ]] || fail "Key mismatch ($test_file_folder/$bucket_file_two, $key_two)" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files "$bucket_file_one" "$bucket_file_two" -} - -@test "test-multipart-upload-from-bucket" { - local bucket_file="bucket-file" - - create_test_files "$bucket_file" || fail "error creating test files" - dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error adding data to test file" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "failed to create bucket: $BUCKET_ONE_NAME" - - multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "error performing multipart upload" - - get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$test_file_folder/$bucket_file-copy" || fail "error getting object" - compare_files "$test_file_folder"/$bucket_file-copy "$test_file_folder"/$bucket_file || fail "data doesn't match" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files $bucket_file -} - -@test "test_multipart_upload_from_bucket_range_too_large" { - local bucket_file="bucket-file" - - create_large_file "$bucket_file" || error creating file "$bucket_file" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - - multipart_upload_from_bucket_range "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 "bytes=0-1000000000" || local upload_result=$? - [[ $upload_result -eq 1 ]] || fail "multipart upload with overly large range should have failed" - log 5 "error: $upload_part_copy_error" - [[ $upload_part_copy_error == *"Range specified is not valid"* ]] || [[ $upload_part_copy_error == *"InvalidRange"* ]] || fail "unexpected error: $upload_part_copy_error" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files $bucket_file -} - -@test "test_multipart_upload_from_bucket_range_valid" { - local bucket_file="bucket-file" - - create_large_file "$bucket_file" || error creating file "$bucket_file" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - - range_max=$((5*1024*1024-1)) - multipart_upload_from_bucket_range "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 "bytes=0-$range_max" || fail "upload failure" - - get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$test_file_folder/$bucket_file-copy" || fail "error retrieving object after upload" - if [[ $(uname) == 'Darwin' ]]; then - object_size=$(stat -f%z "$test_file_folder/$bucket_file-copy") - else - object_size=$(stat --format=%s "$test_file_folder/$bucket_file-copy") - fi - [[ object_size -eq $((range_max*4+4)) ]] || fail "object size mismatch ($object_size, $((range_max*4+4)))" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files $bucket_file -} - -@test "test-presigned-url-utf8-chars" { - test_common_presigned_url_utf8_chars "aws" -} - -@test "test-list-objects-delimiter" { - folder_name="two" - object_name="three" - create_test_folder "$folder_name" || fail "error creating folder" - create_test_files "$folder_name"/"$object_name" || fail "error creating file" - - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket" - - put_object "aws" "$test_file_folder/$folder_name/$object_name" "$BUCKET_ONE_NAME" "$folder_name/$object_name" || fail "failed to add object to bucket" - - list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/" - prefix=$(echo "${objects[@]}" | jq -r ".CommonPrefixes[0].Prefix" 2>&1) || fail "error getting object prefix from object list: $prefix" - [[ $prefix == "$folder_name/" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)" - - list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#" - key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1) || fail "error getting key from object list: $key" - [[ $key == "$folder_name/$object_name" ]] || fail "key doesn't match (expected $key, actual $folder_name/$object_name)" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files $folder_name -} - -#@test "test_put_policy_no_version" { -# policy_file="policy_file" -# -# create_test_files "$policy_file" || fail "error creating policy file" -# -# effect="Allow" -# principal="*" -# action="s3:GetObject" -# resource="arn:aws:s3:::$BUCKET_ONE_NAME/*" -# -# cat < "$test_file_folder"/$policy_file -# { -# "Statement": [ -# { -# "Effect": "$effect", -# "Principal": "$principal", -# "Action": "$action", -# "Resource": "$resource" -# } -# ] -# } -#EOF -# -# setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" -# -# check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" || fail "policy not empty" -# -# put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" -# -# get_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "unable to retrieve policy" -#} - -@test "test_put_policy_invalid_action" { +test_s3api_policy_invalid_action() { policy_file="policy_file" create_test_files "$policy_file" || fail "error creating policy file" @@ -327,20 +14,10 @@ export RUN_USERS=true action="s3:GetObjectt" resource="arn:aws:s3:::$BUCKET_ONE_NAME/*" - cat < "$test_file_folder"/$policy_file - { - "Statement": [ - { - "Effect": "$effect", - "Principal": "$principal", - "Action": "$action", - "Resource": "$resource" - } - ] - } -EOF + # shellcheck disable=SC2154 + setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" || fail "policy not empty" @@ -353,12 +30,13 @@ EOF delete_test_files "$policy_file" } -@test "test_policy_get_object_with_user" { +test_s3api_policy_get_object_with_user() { policy_file="policy_file" username=$USERNAME_ONE password=$PASSWORD_ONE test_file="test_file" + log 5 "username: $USERNAME_ONE, password: $PASSWORD_ONE" create_test_files "$test_file" "$policy_file" || fail "error creating policy file" echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_file" @@ -369,7 +47,7 @@ EOF setup_policy_with_single_statement "$test_file_folder/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" put_object "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object" if ! check_for_empty_policy "s3api" "$BUCKET_ONE_NAME"; then @@ -390,7 +68,7 @@ EOF delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" } -@test "test_policy_get_object_specific_file" { +test_s3api_policy_get_object_specific_file() { policy_file="policy_file" test_file="test_file" test_file_two="test_file_two" @@ -408,7 +86,7 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" @@ -424,7 +102,7 @@ EOF delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" } -@test "test_policy_get_object_file_wildcard" { +test_s3api_policy_get_object_file_wildcard() { policy_file="policy_file_one" policy_file_two="policy_file_two" policy_file_three="policy_fil" @@ -441,7 +119,7 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user account" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" @@ -458,7 +136,7 @@ EOF delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" } -@test "test_policy_get_object_folder_wildcard" { +test_s3api_policy_get_object_folder_wildcard() { policy_file="policy_file" test_folder="test_folder" test_file="test_file" @@ -476,7 +154,7 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" @@ -487,39 +165,20 @@ EOF delete_test_files "$test_folder/$test_file" "$policy_file" } -@test "test_policy_allow_deny" { +test_s3api_policy_allow_deny() { policy_file="policy_file" test_file="test_file" username=$USERNAME_ONE password=$PASSWORD_ONE create_test_files "$policy_file" "$test_file" || fail "error creating policy file" - - principal="$username" - action="s3:GetObject" - resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file" - - cat < "$test_file_folder"/$policy_file - { - "Statement": [ - { - "Effect": "Deny", - "Principal": "$principal", - "Action": "$action", - "Resource": "$resource" - }, - { - "Effect": "Allow", - "Principal": "$principal", - "Action": "$action", - "Resource": "$resource" - } - ] - } -EOF - setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" + + setup_policy_with_double_statement "$test_file_folder/$policy_file" "dummy" \ + "Deny" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file" \ + "Allow" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file" + put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" put_object "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object to bucket" @@ -532,7 +191,7 @@ EOF delete_test_files "$test_file" "$test_file-copy" "$policy_file" } -@test "test_policy_deny" { +test_s3api_policy_deny() { policy_file="policy_file" test_file_one="test_file_one" test_file_two="test_file_two" @@ -540,29 +199,13 @@ EOF password=$PASSWORD_ONE create_test_files "$test_file_one" "$test_file_two" "$policy_file" || fail "error creating policy file, test file" - - cat < "$test_file_folder"/$policy_file -{ - "Statement": [ - { - "Effect": "Deny", - "Principal": "$username", - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file_two" - }, - { - "Effect": "Allow", - "Principal": "$username", - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*" - } - ] -} -EOF - setup_user "$username" "$password" "user" || fail "error creating user" + setup_bucket "s3api" "$BUCKET_ONE_NAME" + + setup_policy_with_double_statement "$test_file_folder/$policy_file" "dummy" \ + "Deny" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file_two" \ + "Allow" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/*" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" log 5 "Policy: $(cat "$test_file_folder/$policy_file")" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" put_object "s3api" "$test_file_folder/$test_file_one" "$BUCKET_ONE_NAME" "$test_file_one" || fail "error copying object one" @@ -576,7 +219,7 @@ EOF delete_test_files "$test_file_one" "$test_file_two" "$test_file_one-copy" "$test_file_two-copy" "$policy_file" } -@test "test_policy_put_wildcard" { +test_s3api_policy_put_wildcard() { policy_file="policy_file" test_folder="test_folder" test_file="test_file" @@ -594,13 +237,14 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" log 5 "Policy: $(cat "$test_file_folder/$policy_file")" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" if put_object_with_user "s3api" "$test_file_folder/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password"; then fail "able to put object despite not being allowed" fi + # shellcheck disable=SC2154 [[ "$put_object_error" == *"Access Denied"* ]] || fail "invalid put object error: $put_object_error" put_object_with_user "s3api" "$test_file_folder/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$username" "$password" || fail "error putting file despite policy permissions" if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_folder/$test_file-copy" "$username" "$password"; then @@ -612,7 +256,7 @@ EOF delete_test_files "$test_folder/$test_file" "$test_file-copy" "$policy_file" } -@test "test_policy_delete" { +test_s3api_policy_delete() { policy_file="policy_file" test_file_one="test_file_one" test_file_two="test_file_two" @@ -630,7 +274,7 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" log 5 "Policy: $(cat "$test_file_folder/$policy_file")" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" @@ -640,13 +284,14 @@ EOF if delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_one" "$username" "$password"; then fail "able to delete object despite lack of permissions" fi + # shellcheck disable=SC2154 [[ "$delete_object_error" == *"Access Denied"* ]] || fail "invalid delete object error: $delete_object_error" delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$username" "$password" || fail "error deleting object despite permissions" delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" delete_test_files "$test_file_one" "$test_file_two" "$policy_file" } -@test "test_policy_get_bucket_policy" { +test_s3api_policy_get_bucket_policy() { policy_file="policy_file" username=$USERNAME_ONE password=$PASSWORD_ONE @@ -660,7 +305,7 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" if get_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then fail "able to retrieve bucket policy despite lack of permissions" @@ -677,14 +322,14 @@ EOF delete_test_files "$policy_file" "$policy_file-copy" } -@test "test_policy_list_multipart_uploads" { +test_s3api_policy_list_multipart_uploads() { policy_file="policy_file" test_file="test_file" username=$USERNAME_ONE password=$PASSWORD_ONE create_test_files "$policy_file" || fail "error creating policy file, test files" - create_large_file "$test_file" || error creating file "$test_file" + create_large_file "$test_file" effect="Allow" principal="$username" @@ -692,10 +337,11 @@ EOF resource="arn:aws:s3:::$BUCKET_ONE_NAME" setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" get_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "error getting bucket policy" log 5 "BUCKET POLICY: $bucket_policy" get_bucket_acl "s3api" "$BUCKET_ONE_NAME" || fail "error getting bucket ACL" + # shellcheck disable=SC2154 log 5 "ACL: $acl" run setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" assert_success "failed to set up policy" @@ -708,6 +354,7 @@ EOF [[ "$list_multipart_uploads_error" == *"Access Denied"* ]] || fail "invalid list multipart uploads error: $list_multipart_uploads_error" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" list_multipart_uploads_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "error listing multipart uploads" + # shellcheck disable=SC2154 log 5 "$uploads" upload_key=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -r ".Uploads[0].Key" 2>&1) || fail "error parsing upload key from uploads message: $upload_key" [[ $upload_key == "$test_file" ]] || fail "upload key doesn't match file marked as being uploaded" @@ -715,7 +362,7 @@ EOF delete_test_files "$policy_file" "$test_file" } -@test "test_policy_put_bucket_policy" { +test_s3api_policy_put_bucket_policy() { policy_file="policy_file" policy_file_two="policy_file_two" username=$USERNAME_ONE @@ -730,7 +377,7 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" if put_bucket_policy_with_user "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" "$username" "$password"; then fail "able to retrieve bucket policy despite lack of permissions" @@ -749,7 +396,7 @@ EOF delete_test_files "$policy_file" "$policy_file_two" "$policy_file-copy" } -@test "test_policy_delete_bucket_policy" { +test_s3api_policy_delete_bucket_policy() { policy_file="policy_file" username=$USERNAME_ONE password=$PASSWORD_ONE @@ -763,7 +410,7 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" if delete_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then fail "able to delete bucket policy with user $username without right permissions" fi @@ -774,7 +421,7 @@ EOF delete_test_files "$policy_file" } -@test "test_policy_get_bucket_acl" { +test_s3api_policy_get_bucket_acl() { policy_file="policy_file" username=$USERNAME_ONE password=$PASSWORD_ONE @@ -788,7 +435,7 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" if get_bucket_acl_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then fail "user able to get bucket ACLs despite permissions" fi @@ -797,89 +444,14 @@ EOF get_bucket_acl_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "error getting bucket ACL despite permissions" } -# ensure that lists of files greater than a size of 1000 (pagination) are returned properly -#@test "test_list_objects_file_count" { -# test_common_list_objects_file_count "aws" -#} - -# ensure that lists of files greater than a size of 1000 (pagination) are returned properly -#@test "test_list_objects_file_count" { -# test_common_list_objects_file_count "aws" -#} - -#@test "test_filename_length" { -# file_name=$(printf "%0.sa" $(seq 1 1025)) -# echo "$file_name" - -# create_test_files "$file_name" || created=$? -# [[ $created -eq 0 ]] || fail "error creating file" - -# setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? -# [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - -# put_object "aws" "$test_file_folder"/"$file_name" "$BUCKET_ONE_NAME"/"$file_name" || local put_object=$? -# [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" -#} - -@test "test_head_bucket" { - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket" - head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info" - log 5 "INFO: $bucket_info" - region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region" - [[ $region != "" ]] || fail "empty bucket region" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" -} - -@test "test_head_bucket_invalid_name" { - if head_bucket "aws" ""; then - fail "able to get bucket info for invalid name" - fi -} - -@test "test_retention_bypass" { - test_retention_bypass_aws_root -} - -@test "test_head_bucket_doesnt_exist" { - setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$? - [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned" - [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info" - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" -} - -@test "test_add_object_metadata" { - object_one="object-one" - test_key="x-test-data" - test_value="test-value" - - create_test_files "$object_one" || fail "error creating test files" - - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket" - - object="$test_file_folder"/"$object_one" - put_object_with_metadata "aws" "$object" "$BUCKET_ONE_NAME" "$object_one" "$test_key" "$test_value" || fail "failed to add object to bucket" - object_exists "aws" "$BUCKET_ONE_NAME" "$object_one" || fail "object not found after being added to bucket" - - get_object_metadata "aws" "$BUCKET_ONE_NAME" "$object_one" || fail "error getting object metadata" - key=$(echo "$metadata" | jq -r 'keys[]' 2>&1) || fail "error getting key from metadata: $key" - value=$(echo "$metadata" | jq -r '.[]' 2>&1) || fail "error getting value from metadata: $value" - [[ $key == "$test_key" ]] || fail "keys doesn't match (expected $key, actual \"$test_key\")" - [[ $value == "$test_value" ]] || fail "values doesn't match (expected $value, actual \"$test_value\")" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files "$object_one" -} - -@test "test_policy_abort_multipart_upload" { +test_s3api_policy_abort_multipart_upload() { policy_file="policy_file" test_file="test_file" username=$USERNAME_ONE create_test_files "$policy_file" || fail "error creating policy file" - create_large_file "$test_file" || fail "error creating large file" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + create_large_file "$test_file" + setup_bucket "s3api" "$BUCKET_ONE_NAME" if [[ $DIRECT == "true" ]]; then setup_user_direct "$username" "user" "$BUCKET_ONE_NAME" || fail "error setting up direct user $username" principal="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/$username\"}" @@ -893,47 +465,20 @@ EOF principal="\"$username\"" fi - cat < "$test_file_folder"/$policy_file -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": $principal, - "Action": "s3:PutObject", - "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*" - }, - { - "Effect": "Deny", - "Principal": $principal, - "Action": "s3:AbortMultipartUpload", - "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*" - } - ] -} -EOF + setup_policy_with_double_statement "$test_file_folder/$policy_file" "2012-10-17" \ + "Allow" "$principal" "s3:PutObject" "arn:aws:s3:::$BUCKET_ONE_NAME/*" \ + "Deny" "$principal" "s3:AbortMultipartUpload" "arn:aws:s3:::$BUCKET_ONE_NAME/*" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting first policy" create_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$username" "$password" || fail "error creating multipart upload" + # shellcheck disable=SC2154 if abort_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$upload_id" "$username" "$password"; then fail "abort multipart upload succeeded despite lack of permissions" fi # shellcheck disable=SC2154 [[ "$abort_multipart_upload_error" == *"AccessDenied"* ]] || fail "unexpected abort error: $abort_multipart_upload_error" - cat < "$test_file_folder"/$policy_file -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": $principal, - "Action": "s3:AbortMultipartUpload", - "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*" - } - ] -} -EOF + setup_policy_with_single_statement "$test_file_folder/$policy_file" "2012-10-17" "Allow" "$principal" "s3:AbortMultipartUpload" "arn:aws:s3:::$BUCKET_ONE_NAME/*" put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" abort_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$upload_id" "$username" "$password" || fail "error aborting multipart upload despite permissions" @@ -942,11 +487,12 @@ EOF delete_test_files "$policy_file" "$test_file" } -@test "test_aws_policy_two_principals" { +test_s3api_policy_two_principals() { policy_file="policy_file" test_file="test_file" - create_test_files "$test_file" "$policy_file" + run create_test_files "$test_file" "$policy_file" + assert_success "error creating test files" run setup_bucket "s3api" "$BUCKET_ONE_NAME" assert_success "error setting up bucket $BUCKET_ONE_NAME" run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user" @@ -954,88 +500,76 @@ EOF run setup_user "$USERNAME_TWO" "$PASSWORD_TWO" "user" assert_success "error setting up user $USERNAME_TWO" - run put_object "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" assert_success "error adding object to bucket" - run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE" + run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE" assert_failure "able to get object with user $USERNAME_ONE despite lack of permission" - run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO" + run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO" assert_failure "able to get object with user $USERNAME_TWO despite lack of permission" - cat < "$test_file_folder"/$policy_file -{ - "Statement": [ - { - "Effect": "Allow", - "Principal": ["$USERNAME_ONE","$USERNAME_TWO"], - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*" - } - ] -} -EOF + run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "Allow" "[\"$USERNAME_ONE\", \"$USERNAME_TWO\"]" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/*" + assert_success "error setting up policy" - run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" + run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" assert_success "error putting policy" - run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE" + run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE" assert_success "error getting object with user $USERNAME_ONE" - run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO" + run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO" assert_success "error getting object with user $USERNAME_TWO" - delete_test_files "$test_file" "$policy_file" "$test_file_folder/copy_one" "$test_file_folder/copy_two" + delete_test_files "$test_file" "$policy_file" "$TEST_FILE_FOLDER/copy_one" "$TEST_FILE_FOLDER/copy_two" delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME" } -@test "test_policy_put_acl" { - if [[ $DIRECT != "true" ]] || [[ $RECREATE_BUCKETS == "false" ]]; then - # https://github.com/versity/versitygw/issues/702 - # https://github.com/versity/versitygw/issues/716 - skip - fi +test_s3api_policy_put_bucket_tagging() { + policy_file="policy_file" + tag_key="TestKey" + tag_value="TestValue" + run create_test_files "$policy_file" + assert_success "error creating test files" + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success "error setting up bucket" + run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user" + assert_success "error setting up user" + + run setup_policy_with_single_statement "$test_file_folder/$policy_file" "2012-10-17" "Allow" "$USERNAME_ONE" "s3:PutBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME" + assert_success "error setting up policy" + run put_bucket_tagging_with_user "$BUCKET_ONE_NAME" "$tag_key" "$tag_value" "$USERNAME_ONE" "$PASSWORD_ONE" + assert_failure "able to put bucket tagging despite lack of permissions" + run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" + assert_success "error putting policy" + run put_bucket_tagging_with_user "$BUCKET_ONE_NAME" "$tag_key" "$tag_value" "$USERNAME_ONE" "$PASSWORD_ONE" + assert_success "unable to put bucket tagging despite user permissions" + + get_and_check_bucket_tags "$BUCKET_ONE_NAME" "$tag_key" "$tag_value" + + delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME" +} + +test_s3api_policy_put_acl() { policy_file="policy_file" test_file="test_file" username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$policy_file" || fail "error creating policy file" - create_large_file "$test_file" || fail "error creating large file" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + create_large_file "$test_file" + setup_bucket "s3api" "$BUCKET_ONE_NAME" put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" || fail "error putting bucket ownership controls" - if [[ $DIRECT == "true" ]]; then - setup_user_direct "$username" "user" "$BUCKET_ONE_NAME" || fail "error setting up direct user $username" - principal="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/$username\"}" - # shellcheck disable=SC2154 - username=$key_id - # shellcheck disable=SC2154 - password=$secret_key - else - password=$PASSWORD_ONE - setup_user "$username" "$password" "user" || fail "error setting up user $username" - principal="\"$username\"" - fi + setup_user "$username" "$password" "user" || fail "error setting up user $username" - cat < "$test_file_folder"/$policy_file -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": $principal, - "Action": "s3:PutBucketAcl", - "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME" - } - ] -} -EOF + setup_policy_with_single_statement "$test_file_folder/$policy_file" "2012-10-17" "Allow" "$username" "s3:PutBucketAcl" "arn:aws:s3:::$BUCKET_ONE_NAME" if [[ $DIRECT == "true" ]]; then put_public_access_block_enable_public_acls "$BUCKET_ONE_NAME" || fail "error enabling public ACLs" fi put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy" - put_bucket_canned_acl_with_user "$BUCKET_ONE_NAME" "public-read" "$username" "$password" || fail "error putting canned acl" + put_bucket_canned_acl_with_user "$BUCKET_ONE_NAME" "public-read" "$username" "$password" || fail "error putting canned acl" get_bucket_acl "s3api" "$BUCKET_ONE_NAME" || fail "error getting bucket acl" # shellcheck disable=SC2154 log 5 "ACL: $acl" @@ -1049,31 +583,38 @@ EOF [[ $uri == "http://acs.amazonaws.com/groups/global/AllUsers" ]] || fail "unexpected URI: $uri" else id=$(echo "$second_grantee" | jq -r ".ID" 2>&1) || fail "error getting ID: $id" - [[ $id == "$username" ]] || fail "unexpected ID: $id" + [[ $id == "all-users" ]] || fail "unexpected ID: $id" fi delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" } -@test "test_put_object_lock_configuration" { - bucket_name=$BUCKET_ONE_NAME - if [[ $RECREATE_BUCKETS == "true" ]]; then - delete_bucket "s3api" "$bucket_name" || fail "error deleting bucket" - create_bucket_object_lock_enabled "$bucket_name" || fail "error setting up bucket" - fi - local enabled="Enabled" - local governance="GOVERNANCE" - local days="1" - put_object_lock_configuration "$bucket_name" "$enabled" "$governance" "$days" || fail "error putting object lock configuration" +test_s3api_policy_get_bucket_tagging() { + policy_file="policy_file" + test_file="test_file" + tag_key="TestKey" + tag_value="TestValue" - get_object_lock_configuration "$bucket_name" || fail "error getting object lock configuration" - log 5 "LOCK CONFIG: $lock_config" - object_lock_configuration=$(echo "$lock_config" | jq -r ".ObjectLockConfiguration" 2>&1) || fail "error getting ObjectLockConfiguration: $object_lock_configuration" - object_lock_enabled=$(echo "$object_lock_configuration" | jq -r ".ObjectLockEnabled" 2>&1) || fail "error getting ObjectLockEnabled: $object_lock_enabled" - [[ $object_lock_enabled == "$enabled" ]] || fail "incorrect ObjectLockEnabled value: $object_lock_enabled" - default_retention=$(echo "$object_lock_configuration" | jq -r ".Rule.DefaultRetention" 2>&1) || fail "error getting DefaultRetention: $default_retention" - mode=$(echo "$default_retention" | jq -r ".Mode" 2>&1) || fail "error getting Mode: $mode" - [[ $mode == "$governance" ]] || fail "incorrect Mode value: $mode" - returned_days=$(echo "$default_retention" | jq -r ".Days" 2>&1) || fail "error getting Days: $returned_days" - [[ $returned_days == "1" ]] || fail "incorrect Days value: $returned_days" - delete_bucket_or_contents "aws" "$bucket_name" -} + run create_test_files "$policy_file" + assert_success "error creating test files" + + setup_bucket "s3api" "$BUCKET_ONE_NAME" + + run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user" + assert_success "error creating user '$USERNAME_ONE'" + + run setup_policy_with_single_statement "$test_file_folder/$policy_file" "2012-10-17" "Allow" "$USERNAME_ONE" "s3:GetBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME" + assert_success "error setting up policy" + + run put_bucket_tagging "s3api" "$BUCKET_ONE_NAME" "$tag_key" "$tag_value" + assert_success "unable to put bucket tagging" + + run get_bucket_tagging_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" + assert_failure "able to get bucket tagging despite lack of permissions" + + run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" + assert_success "error putting policy" + run get_and_check_bucket_tags_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$tag_key" "$tag_value" + assert_success "get and check bucket tags failed" + + delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME" +} \ No newline at end of file diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 07fa9eea..6f52b878 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -95,16 +95,14 @@ export RUN_USERS=true } @test "test_get_bucket_info_s3cmd" { - setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "s3cmd" "$BUCKET_ONE_NAME" head_bucket "s3cmd" "$BUCKET_ONE_NAME" [[ $bucket_info == *"s3://$BUCKET_ONE_NAME"* ]] || fail "failure to retrieve correct bucket info: $bucket_info" delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME" } @test "test_get_bucket_info_doesnt_exist_s3cmd" { - setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "s3cmd" "$BUCKET_ONE_NAME" head_bucket "s3cmd" "$BUCKET_ONE_NAME"a || local info_result=$? [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned" [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info" diff --git a/tests/test_user_aws.sh b/tests/test_user_aws.sh index 2a215c85..f96b1b1d 100755 --- a/tests/test_user_aws.sh +++ b/tests/test_user_aws.sh @@ -37,7 +37,7 @@ export RUN_USERS=true setup_user "$username" "$password" "user" || fail "error creating user if nonexistent" create_test_files "$test_file" || fail "error creating test files" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password"; then fail "able to get object despite not being bucket owner" fi @@ -54,7 +54,7 @@ export RUN_USERS=true setup_user "$username" "$password" "admin" || fail "error creating user if nonexistent" create_test_files "$test_file" || fail "error creating test files" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password"; then fail "able to get object despite not being bucket owner" fi @@ -71,7 +71,7 @@ export RUN_USERS=true setup_user "$username" "$password" "user" || fail "error creating user if nonexistent" create_test_files "$test_file" || fail "error creating test files" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password"; then fail "able to get object despite not being bucket owner" fi @@ -88,7 +88,7 @@ export RUN_USERS=true setup_user "$username" "$password" "admin" || fail "error creating user if nonexistent" create_test_file_with_size "$test_file" 10 || fail "error creating test file" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" put_object_with_user "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password" || fail "failed to add object to bucket" get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password" || fail "error getting object" compare_files "$test_file_folder/$test_file" "$test_file_folder/$test_file-copy" || fail "files don't match" @@ -109,7 +109,7 @@ export RUN_USERS=true setup_user "$username" "$password" "user" || fail "error creating user if nonexistent" create_large_file "$test_file" || fail "error creating test file" - setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" + setup_bucket "s3api" "$BUCKET_ONE_NAME" change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$BUCKET_ONE_NAME" "$username" || fail "error changing bucket ownership" create_multipart_upload_with_user "$BUCKET_ONE_NAME" "dummy" "$username" "$password" || fail "unable to create multipart upload" } diff --git a/tests/test_user_common.sh b/tests/test_user_common.sh index d572651e..8a03fd91 100755 --- a/tests/test_user_common.sh +++ b/tests/test_user_common.sh @@ -23,8 +23,8 @@ test_admin_user() { fi create_user_with_user "$admin_username" "$admin_password" "$user_username" "$user_password" "user" || fail "failed to create user '$user_username'" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket" - delete_bucket "aws" "versity-gwtest-admin-bucket" || fail "error deleting bucket if it exists" + setup_bucket "aws" "$BUCKET_ONE_NAME" + delete_bucket_or_contents_if_exists "aws" "versity-gwtest-admin-bucket" create_bucket_with_user "aws" "versity-gwtest-admin-bucket" "$admin_username" "$admin_password" || fail "error creating bucket with admin user" bucket_one_found=false @@ -46,7 +46,9 @@ test_admin_user() { fi change_bucket_owner "$admin_username" "$admin_password" "versity-gwtest-admin-bucket" "$user_username" || fail "error changing bucket owner" - delete_bucket "aws" "versity-gwtest-admin-bucket" + run delete_bucket "aws" "versity-gwtest-admin-bucket" + assert_success "failed to delete bucket" + delete_user "$user_username" delete_user "$admin_username" } @@ -64,7 +66,6 @@ test_create_user_already_exists() { fail "'user already exists' error not returned" fi - delete_bucket "aws" "versity-gwtest-admin-bucket" delete_user "$username" } @@ -77,8 +78,8 @@ test_user_user() { password="$PASSWORD_ONE" setup_user "$username" "$password" "user" || fail "error setting up user" - delete_bucket "aws" "versity-gwtest-user-bucket" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket '$BUCKET_ONE_NAME'" + delete_bucket_or_contents_if_exists "aws" "versity-gwtest-user-bucket" + setup_bucket "aws" "$BUCKET_ONE_NAME" if create_bucket_with_user "aws" "versity-gwtest-user-bucket" "$username" "$password"; then fail "creating bucket with 'user' account failed to return error" @@ -106,7 +107,8 @@ test_user_user() { fail "user-owned bucket not found in user list" fi - delete_bucket "aws" "versity-gwtest-user-bucket" + run delete_bucket "aws" "versity-gwtest-user-bucket" + assert_success "failed to delete bucket" delete_user "$username" } @@ -118,9 +120,9 @@ test_userplus_operation() { username="$USERNAME_ONE" password="$PASSWORD_ONE" - delete_bucket "aws" "versity-gwtest-userplus-bucket" + delete_bucket_or_contents_if_exists "aws" "versity-gwtest-userplus-bucket" setup_user "$username" "$password" "userplus" || fail "error creating user '$username'" - setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "error setting up bucket '$BUCKET_ONE_NAME'" + setup_bucket "aws" "$BUCKET_ONE_NAME" create_bucket_with_user "aws" "versity-gwtest-userplus-bucket" "$username" "$password" || fail "error creating bucket with user '$username'" @@ -141,6 +143,7 @@ test_userplus_operation() { fail "userplus shouldn't be able to change bucket owner" fi - delete_bucket "aws" "versity-gwtest-admin-bucket" + run delete_bucket "aws" "versity-gwtest-admin-bucket" + assert_success "failed to delete bucket" delete_user "$username" } \ No newline at end of file diff --git a/tests/util.sh b/tests/util.sh index 120d649f..1c2dc1b3 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -13,39 +13,42 @@ source ./tests/commands/delete_object.sh source ./tests/commands/get_bucket_acl.sh source ./tests/commands/get_bucket_ownership_controls.sh source ./tests/commands/get_bucket_tagging.sh +source ./tests/commands/get_object_lock_configuration.sh source ./tests/commands/get_object_tagging.sh source ./tests/commands/head_bucket.sh source ./tests/commands/head_object.sh +source ./tests/commands/list_multipart_uploads.sh source ./tests/commands/list_objects.sh source ./tests/commands/list_parts.sh source ./tests/commands/put_bucket_acl.sh source ./tests/commands/put_bucket_ownership_controls.sh +source ./tests/commands/put_object_legal_hold.sh source ./tests/commands/put_object_lock_configuration.sh source ./tests/commands/upload_part_copy.sh source ./tests/commands/upload_part.sh +source ./tests/util_users.sh # recursively delete an AWS bucket # param: bucket name -# return 0 for success, 1 for failure +# fail if error delete_bucket_recursive() { - if [ $# -ne 2 ]; then - log 2 "delete bucket missing command type, bucket name" - return 1 - fi + log 6 "delete_bucket_recursive" + assert [ $# -eq 2 ] local exit_code=0 local error if [[ $1 == 's3' ]]; then error=$(aws --no-verify-ssl s3 rb s3://"$2" --force 2>&1) || exit_code="$?" elif [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then - delete_bucket_recursive_s3api "$2" || exit_code="$?" + delete_bucket_recursive_s3api "$2" + return 0 elif [[ $1 == "s3cmd" ]]; then error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rb s3://"$2" --recursive 2>&1) || exit_code="$?" elif [[ $1 == "mc" ]]; then error=$(delete_bucket_recursive_mc "$2") || exit_code="$?" else log 2 "invalid command type '$1'" - return 1 + assert [ 1 ] fi if [ $exit_code -ne 0 ]; then @@ -53,7 +56,7 @@ delete_bucket_recursive() { return 0 else log 2 "error deleting bucket recursively: $error" - return 1 + assert [ 1 ] fi fi return 0 @@ -81,100 +84,221 @@ add_governance_bypass_policy() { ] } EOF - put_bucket_policy "s3api" "$1" "$test_file_folder/policy-bypass-governance.txt" || fail "error putting bucket policy" + if ! put_bucket_policy "s3api" "$1" "$test_file_folder/policy-bypass-governance.txt"; then + log 2 "error putting governance bypass policy" + return 1 + fi } -clear_bucket_s3api() { +log_bucket_policy() { + assert [ $# -eq 1 ] + if ! get_bucket_policy "s3api" "$1"; then + log 2 "error getting bucket policy" + return + fi + # shellcheck disable=SC2154 + log 5 "BUCKET POLICY: $bucket_policy" +} + +# param: bucket name +# return 0 for success, 1 for failure +list_and_delete_objects() { + if [ $# -ne 1 ]; then + log 2 "'list_and_delete_objects' missing bucket name" + return 1 + fi if ! list_objects 's3api' "$1"; then - log 2 "error listing objects" + log 2 "error getting object list" return 1 fi # shellcheck disable=SC2154 + log 5 "objects: ${object_array[*]}" for object in "${object_array[@]}"; do - if ! delete_object 's3api' "$1" "$object"; then + if ! clear_object_in_bucket "$1" "$object"; then log 2 "error deleting object $object" - if [[ $delete_object_error == *"WORM"* ]]; then - log 5 "WORM protection found" - if ! put_object_legal_hold "$1" "$object" "OFF"; then - log 2 "error removing object legal hold" - return 1 - fi - sleep 1 - if [[ $LOG_LEVEL_INT -ge 5 ]]; then - if ! get_object_legal_hold "$1" "$object"; then - log 2 "error getting object legal hold status" - return 1 - fi - log 5 "LEGAL HOLD: $legal_hold" - if ! get_object_retention "$1" "$object"; then - log 2 "error getting object retention" - if [[ $get_object_retention_error != *"NoSuchObjectLockConfiguration"* ]]; then - return 1 - fi - fi - log 5 "RETENTION: $retention" - get_bucket_policy "s3api" "$1" || fail "error getting bucket policy" - log 5 "BUCKET POLICY: $bucket_policy" - fi - add_governance_bypass_policy "$1" || fail "error adding governance bypass policy" - if ! delete_object_bypass_retention "$1" "$object" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then - log 2 "error deleting object after legal hold removal" - return 1 - fi - continue - fi return 1 fi done - delete_bucket_policy "s3api" "$1" || fail "error deleting bucket policy" - # TODO uncomment after #716 is fixed - #reset_bucket_acl "$1" || fail "error resetting bucket ACLs" - put_object_lock_configuration_disabled "$1" || fail "error removing object lock config" - #change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID" || fail "error changing bucket owner" } -delete_bucket_recursive_s3api() { - if [[ $# -ne 1 ]]; then - log 2 "delete bucket recursive command for s3api requires bucket name" +# param: bucket name +# return 0 for success, 1 for failure +check_ownership_rule_and_reset_acl() { + if [ $# -ne 1 ]; then + log 2 "'check_ownership_rule_and_reset_acl' requires bucket name" + return 1 + fi + if ! get_bucket_ownership_controls "$1"; then + log 2 "error getting bucket ownership controls" + return 1 + fi + # shellcheck disable=SC2154 + if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then + log 2 "error getting object ownership rule: $object_ownership_rule" + return 1 + fi + if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then + log 2 "error resetting bucket ACL" + return 1 + fi +} + +# param: bucket name +# return 0 for success, 1 for error +check_and_disable_object_lock_config() { + if [ $# -ne 1 ]; then + log 2 "'check_and_disable_object_lock_config' requires bucket name" return 1 fi - clear_bucket_s3api "$1" || fail "error clearing bucket" - - delete_bucket 's3api' "$1" || local delete_bucket_result=$? - if [[ $delete_bucket_result -ne 0 ]]; then - log 2 "error deleting bucket" + local lock_config_exists=true + if ! get_object_lock_configuration "$1"; then + # shellcheck disable=SC2154 + if [[ "$get_object_lock_config_err" == *"does not exist"* ]]; then + lock_config_exists=false + else + log 2 "error getting object lock config" + return 1 + fi + fi + if [[ $lock_config_exists == true ]] && ! put_object_lock_configuration_disabled "$1"; then + log 2 "error disabling object lock config" return 1 fi +} + +# restore bucket to pre-test state (or prep for deletion) +# param: bucket name +# fail on error +clear_bucket_s3api() { + log 6 "clear_bucket_s3api" + + assert [ $# -eq 1 ] + + if [[ $LOG_LEVEL_INT -ge 5 ]]; then + run log_bucket_policy "$1" + assert_success "error logging bucket policy" + fi + + run list_and_delete_objects "$1" + assert_success "error listing and delete objects" + + run delete_bucket_policy "s3api" "$1" + assert_success "error deleting bucket policy" + + #run check_ownership_rule_and_reset_acl "$1" + #assert_success "error checking ownership rule and resetting acl" + + run check_and_disable_object_lock_config "$1" + assert_success "error checking and disabling object lock config" + + #if ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID"; then + # log 2 "error changing bucket owner back to root" + # return 1 + #fi +} + +# params: bucket, object name +# return 0 for success, 1 for error +clear_object_in_bucket() { + log 6 "clear_object_in_bucket" + if [ $# -ne 2 ]; then + log 2 "'clear_object_in_bucket' requires bucket, object name" + return 1 + fi + if ! delete_object 's3api' "$1" "$2"; then + # shellcheck disable=SC2154 + log 2 "error deleting object $2: $delete_object_error" + if ! check_for_and_remove_worm_protection "$1" "$2" "$delete_object_error"; then + log 2 "error checking for and removing worm protection if needed" + return 1 + fi + fi return 0 } -# delete contents of a bucket -# param: command type, bucket name -# return 0 for success, 1 for failure -delete_bucket_contents() { - if [ $# -ne 2 ]; then - log 2 "delete bucket missing command id, bucket name" +# params: bucket, object, possible WORM error after deletion attempt +# return 0 for success, 1 for error +check_for_and_remove_worm_protection() { + if [ $# -ne 3 ]; then + log 2 "'check_for_and_remove_worm_protection' command requires bucket, object, error" return 1 fi + if [[ $3 == *"WORM"* ]]; then + log 5 "WORM protection found" + if ! put_object_legal_hold "$1" "$2" "OFF"; then + log 2 "error removing object legal hold" + return 1 + fi + sleep 1 + if [[ $LOG_LEVEL_INT -ge 5 ]]; then + log_worm_protection "$1" "$2" + fi + if ! add_governance_bypass_policy "$1"; then + log 2 "error adding new governance bypass policy" + return 1 + fi + if ! delete_object_bypass_retention "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then + log 2 "error deleting object after legal hold removal" + return 1 + fi + fi + return 0 +} + +# params: bucket name, object +log_worm_protection() { + if ! get_object_legal_hold "$1" "$2"; then + log 2 "error getting object legal hold status" + return + fi + # shellcheck disable=SC2154 + log 5 "LEGAL HOLD: $legal_hold" + if ! get_object_retention "$1" "$2"; then + log 2 "error getting object retention" + # shellcheck disable=SC2154 + if [[ $get_object_retention_error != *"NoSuchObjectLockConfiguration"* ]]; then + return + fi + fi + # shellcheck disable=SC2154 + log 5 "RETENTION: $retention" +} + +# params: bucket name +# fail if unable to delete bucket +delete_bucket_recursive_s3api() { + log 6 "delete_bucket_recursive_s3api" + assert [ $# -eq 1 ] + + clear_bucket_s3api "$1" + + run delete_bucket 's3api' "$1" + assert_success "error deleting bucket" + + return 0 +} + +# params: client, bucket name +# fail if error +delete_bucket_contents() { + log 6 "delete_bucket_contents" + assert [ $# -eq 2 ] + local exit_code=0 local error if [[ $1 == "aws" ]] || [[ $1 == 's3api' ]]; then - clear_bucket_s3api "$2" || exit_code="$?" + clear_bucket_s3api "$2" + return 0 elif [[ $1 == "s3cmd" ]]; then - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate del s3://"$2" --recursive --force 2>&1) || exit_code="$?" + delete_bucket_recursive "s3cmd" "$1" + return 0 elif [[ $1 == "mc" ]]; then - error=$(mc --insecure rm --force --recursive "$MC_ALIAS"/"$2" 2>&1) || exit_code="$?" - else - log 2 "invalid command type $1" - return 1 + delete_bucket_recursive "mc" "$1" + return 0 fi - if [ $exit_code -ne 0 ]; then - log 2 "error deleting bucket contents: $error" - return 1 - fi - return 0 + assert [ 1 ] } # check if bucket exists @@ -182,142 +306,153 @@ delete_bucket_contents() { # return 0 for true, 1 for false, 2 for error bucket_exists() { if [ $# -ne 2 ]; then - log 2 "bucket exists check missing command type, bucket name" + log 2 "bucket_exists command requires client, bucket name" return 2 fi - - if ! head_bucket "$1" "$2"; then - # shellcheck disable=SC2154 - bucket_info=$(echo "$bucket_info" | grep -v "InsecureRequestWarning") - log 5 "$bucket_info" - if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then - log 5 "bucket not found" - return 1 - fi - log 2 "error checking if bucket exists" + local exists=0 + head_bucket "$1" "$2" || exists=$? + # shellcheck disable=SC2181 + if [ $exists -ne 0 ] && [ $exists -ne 1 ]; then + log 2 "unexpected error checking if bucket exists" return 2 fi - return 0 + if [ $exists -eq 0 ]; then + return 0 + fi + return 1 } +# param: bucket name +# return 0 for success, 1 for error abort_all_multipart_uploads() { - assert [ $# -eq 1 ] - run aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" + if [ $# -ne 1 ]; then + log 2 "'abort_all_multipart_uploads' requires bucket name" + return 1 + fi + if ! list_multipart_uploads "$1"; then + log 2 "error listing multipart uploads" + return 1 + fi # shellcheck disable=SC2154 - assert_success "error listing uploads: $output" - log 5 "UPLOADS: $output" - if ! upload_set=$(echo "$output" | grep -v "InsecureRequestWarning" | jq -c '.Uploads[]' 2>&1); then + log 5 "UPLOADS: $uploads" + if ! upload_set=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -c '.Uploads[]' 2>&1); then if [[ $upload_set == *"Cannot iterate over null"* ]]; then return 0 + else + log 2 "error getting upload set: $upload_set" + return 1 fi - fail "error getting upload set: $upload_set" fi log 5 "UPLOAD SET: $upload_set" for upload in $upload_set; do log 5 "UPLOAD: $upload" - upload_id=$(echo "$upload" | jq -r ".UploadId" 2>&1) - assert [ $? -eq 0 ] + if ! upload_id=$(echo "$upload" | jq -r ".UploadId" 2>&1); then + log 2 "error getting upload ID: $upload_id" + return 1 + fi log 5 "upload ID: $upload_id" - key=$(echo "$upload" | jq -r ".Key" 2>&1) - assert [ $? -eq 0 ] - log 5 "Key: $key" - + if ! key=$(echo "$upload" | jq -r ".Key" 2>&1); then + log 2 "error getting key: $key" + return 1 + fi log 5 "Aborting multipart upload for key: $key, UploadId: $upload_id" - run aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$key" --upload-id "$upload_id" - assert_success "error aborting upload: $output" + if ! abort_multipart_upload "$1" "$key" "$upload_id"; then + log 2 "error aborting multipart upload" + return 1 + fi done } -# delete buckets or just the contents depending on RECREATE_BUCKETS parameter -# params: command type, bucket name -# return: 0 for success, 1 for failure -delete_bucket_or_contents() { - if [ $# -ne 2 ]; then - log 2 "delete bucket or contents function requires command type, bucket name" +# param: bucket name +# return 1 for failure, 0 for success +get_object_ownership_rule_and_update_acl() { + if [ $# -ne 1 ]; then + log 2 "'get_object_ownership_rule_and_update_acl' requires bucket name" return 1 fi + if ! get_object_ownership_rule "$1"; then + log 2 "error getting object ownership rule" + return 1 + fi + log 5 "object ownership rule: $object_ownership_rule" + if [[ "$object_ownership_rule" != "BucketOwnerEnforced" ]] && ! put_bucket_canned_acl "$1" "private"; then + log 2 "error resetting bucket ACLs" + return 1 + fi +} + +# params: client, bucket name +# fail if error +delete_bucket_or_contents() { + log 6 "delete_bucket_or_contents" + assert [ $# -eq 2 ] if [[ $RECREATE_BUCKETS == "false" ]]; then - if ! delete_bucket_contents "$1" "$2"; then - log 2 "error deleting bucket contents" - return 1 - fi - if ! delete_bucket_policy "$1" "$2"; then - log 2 "error deleting bucket policies" - return 1 - fi - if ! get_object_ownership_rule "$2"; then - log 2 "error getting object ownership rule" - return 1 - fi - log 5 "object ownership rule: $object_ownership_rule" - if [[ "$object_ownership_rule" != "BucketOwnerEnforced" ]] && ! put_bucket_canned_acl "$2" "private"; then - log 2 "error resetting bucket ACLs" - return 1 - fi + delete_bucket_contents "$1" "$2" + + run delete_bucket_policy "$1" "$2" + assert_success "error deleting bucket policies" + + run get_object_ownership_rule_and_update_acl "$2" + assert_success "error getting object ownership rule and updating acl" + run abort_all_multipart_uploads "$2" assert_success "error aborting multipart uploads" + log 5 "bucket contents, policy, ACL deletion success" return 0 fi - if ! delete_bucket_recursive "$1" "$2"; then - log 2 "Bucket deletion error" - return 1 - fi + run delete_bucket_recursive "$1" "$2" + assert_success "error with recursive bucket delete" log 5 "bucket deletion success" return 0 } +# params: client, bucket name +# fail if unable to delete bucket (RECREATE_BUCKETS=true) or contents (RECREATE_BUCKETS=false) delete_bucket_or_contents_if_exists() { - if [ $# -ne 2 ]; then - log 2 "bucket creation function requires command type, bucket name" - return 1 - fi - local bucket_exists_result - bucket_exists "$1" "$2" || local bucket_exists_result=$? - if [[ $bucket_exists_result -eq 2 ]]; then - log 2 "Bucket existence check error" - return 1 - fi - if [[ $bucket_exists_result -eq 0 ]]; then - if ! delete_bucket_or_contents "$1" "$2"; then - log 2 "error deleting bucket or contents" - return 1 - fi + log 6 "delete_bucket_or_contents_if_exists" + + assert [ $# -eq 2 ] + + if bucket_exists "$1" "$2"; then + delete_bucket_or_contents "$1" "$2" log 5 "bucket and/or bucket data deletion success" return 0 fi if [[ $RECREATE_BUCKETS == "false" ]]; then log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user" - return 1 + assert [ 1 ] fi return 0 } -# if RECREATE_BUCKETS is set to true create bucket, deleting it if it exists to clear state. If not, -# check to see if it exists and return an error if it does not. -# param: bucket name -# return 0 for success, 1 for failure +# params: client, bucket name +# fail if bucket is not properly set up setup_bucket() { + log 6 "setup_bucket" + assert [ $# -eq 2 ] + if [[ $1 == "s3cmd" ]]; then log 5 "putting bucket ownership controls" - put_bucket_ownership_controls "$2" "BucketOwnerPreferred" + if bucket_exists "s3cmd" "$2"; then + run put_bucket_ownership_controls "$2" "BucketOwnerPreferred" + assert_success "error putting bucket ownership controls" + fi fi - if ! delete_bucket_or_contents_if_exists "$1" "$2"; then - log 2 "error deleting bucket, or checking for bucket existence" - return 1 - fi - local create_result + + delete_bucket_or_contents_if_exists "$1" "$2" + log 5 "util.setup_bucket: command type: $1, bucket name: $2" if [[ $RECREATE_BUCKETS == "true" ]]; then - if ! create_bucket "$1" "$2"; then - log 2 "Error creating bucket" - return 1 - fi + run create_bucket "$1" "$2" + assert_success "error creating bucket" log 5 "bucket creation success" + if [[ $1 == "s3cmd" ]]; then log 5 "putting bucket ownership controls" - put_bucket_ownership_controls "$2" "BucketOwnerPreferred" || fail "putting bucket ownership controls failed" + run put_bucket_ownership_controls "$2" "BucketOwnerPreferred" + assert_success "error putting bucket ownership controls" fi else log 5 "skipping bucket re-creation" diff --git a/tests/util_file.sh b/tests/util_file.sh index 9b328fba..5b45b6c9 100644 --- a/tests/util_file.sh +++ b/tests/util_file.sh @@ -3,24 +3,47 @@ source ./tests/logger.sh # create a test file and export folder. do so in temp folder -# params: filename -# export test file folder on success, return 1 for error +# params: filenames +# return 0 for success, 1 for failure create_test_files() { - assert [ $# -gt 0 ] - test_file_folder=$PWD + log 6 "create_test_files" + if [ $# -lt 1 ]; then + log 2 "'create_test_files' requires minimum of one file name" + return 1 + fi + #test_file_folder=$PWD if [[ -z "$GITHUB_ACTIONS" ]]; then - create_test_file_folder + if ! create_test_file_folder; then + log 2 "error creating test file folder" + return 1 + fi fi for name in "$@"; do - if [[ -e "$test_file_folder/$name" ]]; then - run rm "$test_file_folder/$name" - # shellcheck disable=SC2154 - assert_success "error removing existing test file: $output" + if ! create_test_file "$name"; then + log 2 "error creating test file" + return 1 fi - run touch "$test_file_folder"/"$name" - assert_success "error creating new file: $output" done - export test_file_folder + #export test_file_folder + return 0 +} + +create_test_file() { + if [ $# -ne 1 ]; then + log 2 "'create_test_file' requires name" + return 1 + fi + if [[ -e "$TEST_FILE_FOLDER/$name" ]]; then + if ! error=$(rm "$TEST_FILE_FOLDER/$name" 2>&1); then + log 2 "error removing old test file: $error" + return 1 + fi + fi + if ! error=$(touch "$TEST_FILE_FOLDER/$name"); then + log 2 "error creating new test file: $error" + return 1 + fi + return 0 } create_test_file_with_size() { @@ -32,7 +55,7 @@ create_test_file_with_size() { log 2 "error creating test file" return 1 fi - if ! error=$(dd if=/dev/urandom of="$test_file_folder"/"$1" bs=1 count="$2" 2>&1); then + if ! error=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER"/"$1" bs=1 count="$2" 2>&1); then log 2 "error writing file data: $error" return 1 fi @@ -41,19 +64,23 @@ create_test_file_with_size() { create_test_folder() { if [ $# -lt 1 ]; then - echo "create test folder command missing folder name" + log 2 "'create_test_folder' command requires at least one folder" return 1 fi - test_file_folder=$PWD + #test_file_folder=$PWD if [[ -z "$GITHUB_ACTIONS" ]]; then - create_test_file_folder + if ! create_test_file_folder; then + log 2 "error creating test file folder" + return 1 + fi fi for name in "$@"; do - mkdir -p "$test_file_folder"/"$name" || local mkdir_result=$? - if [[ $mkdir_result -ne 0 ]]; then - echo "error creating file $name" + if ! error=$(run mkdir -p "$TEST_FILE_FOLDER"/"$name" 2>&1); then + log 2 "error creating test folder $name: $error" + return 1 fi done + return 0 } # delete a test file @@ -64,12 +91,12 @@ delete_test_files() { echo "delete test files command missing filenames" return 1 fi - if [ -z "$test_file_folder" ]; then + if [ -z "$TEST_FILE_FOLDER" ]; then echo "no test file folder defined, not deleting" return 1 fi for name in "$@"; do - rm -rf "${test_file_folder:?}"/"${name:?}" || rm_result=$? + rm -rf "${TEST_FILE_FOLDER:?}"/"${name:?}" || rm_result=$? if [[ $rm_result -ne 0 ]]; then echo "error deleting file $name" fi @@ -120,38 +147,41 @@ compare_files() { return 1 } +# return 0 on success, 1 on failure create_test_file_folder() { - if [[ -n $TMPDIR ]]; then - test_file_folder=${TMPDIR}versity-gwtest - else - test_file_folder=$PWD/versity-gwtest - fi - if ! error=$(mkdir -p "$test_file_folder" 2>&1); then + log 6 "create_test_file_folder" + if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then # shellcheck disable=SC2035 - run [[ "$error" == *"File exists"* ]] - assert_success "error creating test file folder: $error" + if [[ "$error" != *"File exists"* ]]; then + log 2 "error creating test file folder: $error" + return 1 + fi fi - export test_file_folder + export test_file_folder=$TEST_FILE_FOLDER + return 0 } -# generate 16MB file +# generate 160MB file # input: filename -# return 0 for success, 1 for error +# return 0 on success, 1 on failure create_large_file() { - if [[ $# -ne 1 ]]; then - echo "generate large file function requires filename" + log 6 "create_large_file" + if [ $# -ne 1 ]; then + log 2 "'create_large_file' requires filename" return 1 fi - test_file_folder=$PWD + #test_file_folder=$PWD/versity-gwtest-files if [[ -z "$GITHUB_ACTIONS" ]]; then - create_test_file_folder + if ! create_test_file_folder; then + log 2 "error creating test file" + return 1 + fi fi filesize=$((160*1024*1024)) - error=$(dd if=/dev/urandom of="$test_file_folder"/"$1" bs=1024 count=$((filesize/1024))) || dd_result=$? - if [[ $dd_result -ne 0 ]]; then - echo "error creating file: $error" + if ! error=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER"/"$1" bs=1024 count=$((filesize/1024)) 2>&1); then + log 2 "error creating large file: $error" return 1 fi return 0 @@ -162,13 +192,13 @@ create_test_file_count() { echo "create test file count function missing bucket name, count" return 1 fi - test_file_folder=$PWD + #test_file_folder=$PWD if [[ -z "$GITHUB_ACTIONS" ]]; then create_test_file_folder fi local touch_result for ((i=1;i<=$1;i++)) { - error=$(touch "$test_file_folder/file_$i") || touch_result=$? + error=$(touch "$TEST_FILE_FOLDER/file_$i") || touch_result=$? if [[ $touch_result -ne 0 ]]; then echo "error creating file_$i: $error" return 1 @@ -176,7 +206,7 @@ create_test_file_count() { } # shellcheck disable=SC2153 if [[ $LOG_LEVEL -ge 5 ]]; then - ls_result=$(ls "$test_file_folder"/file_*) + ls_result=$(ls "$TEST_FILE_FOLDER/file_*") log 5 "$ls_result" fi return 0 diff --git a/tests/util_lock_config.sh b/tests/util_lock_config.sh new file mode 100644 index 00000000..e3413bbf --- /dev/null +++ b/tests/util_lock_config.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# params: bucket name, expected enabled value, expected governance mode, expected days +# return 0 for success, 1 for failure +get_and_check_object_lock_config() { + if [ $# -ne 4 ]; then + log 2 "'get_and_check_lock_config' requires bucket name, expected enabled value, expected governance mode, expected days" + return 1 + fi + + if ! get_object_lock_configuration "$1"; then + log 2 "error getting object lock config" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "LOCK CONFIG: $lock_config" + if ! object_lock_configuration=$(echo "$lock_config" | grep -v "InsecureRequestWarning" | jq -r ".ObjectLockConfiguration" 2>&1); then + log 2 "error getting ObjectLockConfiguration: $object_lock_configuration" + return 1 + fi + if ! object_lock_enabled=$(echo "$object_lock_configuration" | jq -r ".ObjectLockEnabled" 2>&1); then + log 2 "error getting object lock enabled status: $object_lock_enabled" + return 1 + fi + if [[ $object_lock_enabled != "$2" ]]; then + log 2 "incorrect ObjectLockEnabled value: $object_lock_enabled" + return 1 + fi + if ! default_retention=$(echo "$object_lock_configuration" | jq -r ".Rule.DefaultRetention" 2>&1); then + log 2 "error getting DefaultRetention: $default_retention" + return 1 + fi + if ! mode=$(echo "$default_retention" | jq -r ".Mode" 2>&1); then + log 2 "error getting Mode: $mode" + return 1 + fi + if [[ $mode != "$3" ]]; then + log 2 "incorrect Mode value: $mode" + return 1 + fi + if ! returned_days=$(echo "$default_retention" | jq -r ".Days" 2>&1); then + log 2 "error getting Days: $returned_days" + return 1 + fi + if [[ $returned_days != "$4" ]]; then + log 2 "incorrect Days value: $returned_days" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/util_policy.sh b/tests/util_policy.sh index c52696b8..5dad18d8 100644 --- a/tests/util_policy.sh +++ b/tests/util_policy.sh @@ -28,22 +28,73 @@ check_for_empty_policy() { return 0 } -setup_policy_with_single_statement() { - if [[ $# -ne 6 ]]; then - "'setup single policy' command requires file, version, effect, principal, action, resource" +get_modified_principal() { + log 6 "get_modified_principal" + assert [ $# -eq 1 ] + local first_char="${1:0:1}" + if [ "$first_char" != '{' ] && [ "$first_char" != '[' ] && [ "$first_char" != '"' ]; then + # shellcheck disable=SC2089 + modified_principal="\"$1\"" + else + modified_principal=$1 fi - cat < "$1" + export modified_principal +} + +# params: file, version, effect, principal, action, resource +# fail on error +setup_policy_with_single_statement() { + log 6 "setup_policy_with_single_statement" + assert [ $# -eq 6 ] + log 5 "policy file: $1" + get_modified_principal "$4" + bash -c "cat < $1 { - "Version": "$2", - "Statement": [ + \"Version\": \"$2\", + \"Statement\": [ { - "Effect": "$3", - "Principal": "$4", - "Action": "$5", - "Resource": "$6" + \"Effect\": \"$3\", + \"Principal\": $modified_principal, + \"Action\": \"$5\", + \"Resource\": \"$6\" } ] } -EOF -log 5 "$(cat "$1")" +EOF" + # shellcheck disable=SC2154 + #assert_success "failed to set up policy: $output" + log 5 "policy data: $(cat "$1")" } + +# params: file, version, two sets: effect, principal, action, resource +# fail on error +setup_policy_with_double_statement() { + log 6 "setup_policy_with_double_statement" + assert [ $# -eq 10 ] + get_modified_principal "$4" + principal_one=$modified_principal + get_modified_principal "$8" + principal_two=$modified_principal + run bash -c "cat < $1 +{ + \"Version\": \"$2\", + \"Statement\": [ + { + \"Effect\": \"$3\", + \"Principal\": $principal_one, + \"Action\": \"$5\", + \"Resource\": \"$6\" + }, + { + \"Effect\": \"$7\", + \"Principal\": $principal_two, + \"Action\": \"$9\", + \"Resource\": \"${10}\" + } + ] +} +EOF" + # shellcheck disable=SC2154 + assert_success "failed to set up policy: $output" + log 5 "policy data: $(cat "$1")" +} \ No newline at end of file diff --git a/tests/util_tags.sh b/tests/util_tags.sh new file mode 100644 index 00000000..4b8dcc89 --- /dev/null +++ b/tests/util_tags.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# params: username, password, bucket, expected key, expected value +# return 0 for success, 1 for failure +get_and_check_bucket_tags_with_user() { + log 6 "get_and_check_bucket_tags" + if [ $# -ne 5 ]; then + log 2 "'get_and_check_bucket_tags' requires username, password, bucket, expected key, expected value" + return 1 + fi + if ! get_bucket_tagging_with_user "$1" "$2" "$3"; then + log 2 "error retrieving bucket tagging" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "TAGS: $tags" + if ! tag=$(echo "$tags" | jq -r ".TagSet[0]" 2>&1); then + log 2 "error getting tag: $tag" + return 1 + fi + if ! key=$(echo "$tag" | jq -r ".Key" 2>&1); then + log 2 "error getting key: $key" + return 1 + fi + if [ "$key" != "$4" ]; then + log 2 "key mismatch ($key, $4)" + return 1 + fi + if ! value=$(echo "$tag" | jq -r ".Value" 2>&1); then + log 2 "error getting value: $value" + return 1 + fi + if [ "$value" != "$5" ]; then + log 2 "value mismatch ($value, $5)" + return 1 + fi + return 0 +} + +# params: bucket, expected tag key, expected tag value +# fail on error +get_and_check_bucket_tags() { + assert [ $# -eq 3 ] + run get_and_check_bucket_tags_with_user "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$2" "$3" + assert_success "error getting and checking bucket tags" + return 0 +} diff --git a/tests/util_users.sh b/tests/util_users.sh index 65907384..9cb5b00d 100644 --- a/tests/util_users.sh +++ b/tests/util_users.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash setup_user() { + log 6 "setup_user" if [[ $# -ne 3 ]]; then log 2 "'setup user' command requires username, password, and role" return 1 @@ -19,6 +20,8 @@ setup_user() { } setup_user_direct() { + log 6 "setup_user_direct" + log 5 "username: $1, role: $2, bucket: $3" if [[ $# -ne 3 ]]; then log 2 "'setup user direct' command requires username, role, and bucket" return 1 @@ -37,6 +40,7 @@ setup_user_direct() { } create_user_versitygw() { + log 6 "create_user_versitygw" if [[ $# -ne 3 ]]; then log 2 "create user command requires user ID, key, and role" return 1 @@ -49,6 +53,7 @@ create_user_versitygw() { } create_user_if_nonexistent() { + log 6 "create_user_if_nonexistent" if [[ $# -ne 3 ]]; then echo "create user command requires user ID, key, and role" return 1 @@ -62,6 +67,7 @@ create_user_if_nonexistent() { } put_user_policy_userplus() { + log 6 "put_user_policy_userplus" if [[ $# -ne 1 ]]; then log 2 "'put user policy userplus' function requires username" return 1 @@ -106,6 +112,7 @@ EOF } put_user_policy() { + log 6 "put_user_policy" if [[ $# -ne 3 ]]; then log 2 "attaching user policy requires user ID, role, bucket name" return 1 @@ -129,6 +136,7 @@ put_user_policy() { } create_user_direct() { + log 6 "create_user_direct" if [[ $# -ne 3 ]]; then log 2 "create user direct command requires desired username, role, bucket name" return 1 @@ -157,6 +165,7 @@ create_user_direct() { } create_user_with_user() { + log 6 "create_user_with_user" if [[ $# -ne 5 ]]; then log 2 "create user with user command requires creator ID, key, and new user ID, key, and role" return 1 @@ -169,6 +178,7 @@ create_user_with_user() { } list_users_direct() { + log 6 "list_users_direct" # AWS_ENDPOINT_URL of s3.amazonaws.com doesn't work here if ! users=$(aws --profile="$AWS_PROFILE" iam list-users 2>&1); then log 2 "error listing users via direct s3 call: $users" @@ -188,6 +198,7 @@ list_users_direct() { } list_users() { + log 6 "list_users" if [[ $DIRECT == "true" ]]; then if ! list_users_direct; then log 2 "error listing users via direct s3 call" @@ -203,6 +214,7 @@ list_users() { } list_users_versitygw() { + log 6 "list_users_versitygw" users=$($VERSITY_EXE admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users) || local list_result=$? if [[ $list_result -ne 0 ]]; then echo "error listing users: $users" @@ -217,6 +229,7 @@ list_users_versitygw() { } user_exists() { + log 6 "user_exists" if [[ $# -ne 1 ]]; then log 2 "user exists command requires username" return 2 @@ -235,6 +248,7 @@ user_exists() { } delete_user_direct() { + log 6 "delete_user_direct" if [[ $# -ne 1 ]]; then log 2 "delete user direct command requires username" return 1 @@ -271,6 +285,7 @@ delete_user_direct() { } delete_user_versitygw() { + log 6 "delete_user_versitygw" if [[ $# -ne 1 ]]; then log 2 "delete user via versitygw command requires user ID or username" return 1 @@ -285,6 +300,7 @@ delete_user_versitygw() { } delete_user() { + log 6 "delete_user" if [[ $# -ne 1 ]]; then log 2 "delete user command requires user ID" return 1 @@ -304,6 +320,7 @@ delete_user() { } change_bucket_owner_direct() { + log 6 "change_bucket_owner_direct" if [[ $# -ne 4 ]]; then echo "change bucket owner command requires ID, key, bucket name, and new owner" return 1 @@ -312,6 +329,7 @@ change_bucket_owner_direct() { } change_bucket_owner() { + log 6 "change_bucket_owner" if [[ $# -ne 4 ]]; then echo "change bucket owner command requires ID, key, bucket name, and new owner" return 1 @@ -321,7 +339,9 @@ change_bucket_owner() { log 2 "error changing bucket owner direct to s3" return 1 fi + return 0 fi + log 5 "changing owner for bucket $3, new owner: $4" error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" change-bucket-owner --bucket "$3" --owner "$4" 2>&1) || local change_result=$? if [[ $change_result -ne 0 ]]; then echo "error changing bucket owner: $error" @@ -331,6 +351,7 @@ change_bucket_owner() { } get_bucket_owner() { + log 6 "get_bucket_owner" if [[ $# -ne 1 ]]; then log 2 "'get bucket owner' command requires bucket name" return 1