From 0facfdc9fd8ad029d84345f3dd42e7c6bc2b6323 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Fri, 2 Aug 2024 20:24:37 -0300 Subject: [PATCH] test: multiple policy principals, improved bucket cleanup, general cleanup --- .github/workflows/system.yml | 6 ++ tests/.env.default | 9 +- tests/README.md | 17 +-- tests/commands/put_bucket_acl.sh | 33 ++++++ tests/env.sh | 172 +++++++++++------------------ tests/report.sh | 5 + tests/setup.sh | 20 ++-- tests/teardown_static.sh | 19 +++- tests/test_aws.sh | 179 +++++++++++++++---------------- tests/test_aws_root_inner.sh | 24 +++-- tests/test_common.sh | 8 +- tests/test_user_aws.sh | 20 ++-- tests/test_user_common.sh | 20 ++-- tests/util.sh | 38 ++++++- tests/util_file.sh | 19 ++-- tests/versity.sh | 5 +- 16 files changed, 322 insertions(+), 272 deletions(-) diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 481a6eb5..d1a5dae5 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -103,6 +103,8 @@ jobs: run: | git clone https://github.com/bats-core/bats-core.git cd bats-core && ./install.sh $HOME + git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support + git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert - name: Install s3cmd run: | @@ -135,6 +137,10 @@ jobs: MC_ALIAS: versity LOG_LEVEL: 4 GOCOVERDIR: ${{ github.workspace }}/cover + USERNAME_ONE: ABCDEFG + PASSWORD_ONE: 1234567 + USERNAME_TWO: HIJKLMN + PASSWORD_TWO: 8901234 run: | make testbin export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST diff --git a/tests/.env.default b/tests/.env.default index c856e312..bb689bfb 100644 --- a/tests/.env.default +++ b/tests/.env.default @@ -17,4 +17,11 @@ GOCOVERDIR=$PWD/cover USERS_FOLDER=$PWD/iam #TEST_LOG_FILE=test.log #VERSITY_LOG_FILE=versity.log -IAM_TYPE=folder \ No newline at end of file +IAM_TYPE=folder +DIRECT=false +#DIRECT_DISPLAY_NAME= +#COVERAGE_DB=coverage.sql +USERNAME_ONE=ABCDEFG +PASSWORD_ONE=HIJKLMN +USERNAME_TWO=HIJKLMN +PASSWORD_TWO=OPQRSTU diff --git a/tests/README.md b/tests/README.md index f9c4a6bb..dc085d27 100644 --- a/tests/README.md +++ b/tests/README.md @@ -9,10 +9,11 @@ * **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). * **s3cmd**: Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md). * **mc**: Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html). -3. Install BATS. Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html). -4. If running on Mac OS, install **jq** with the command `brew install jq`. -4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file. -5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile. Example: +3. Install **BATS**. Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html). +4. Install **bats-support** and **bats-assert**. This can be done by saving the root folder of each repo (both located in https://github.com/bats-core/) in the `tests` folder. +5. If running on Mac OS, install **jq** with the command `brew install jq`. +6. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file. +7. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile. Example: ``` export AWS_PROFILE=versity-test export AWS_ACCESS_KEY_ID= @@ -22,14 +23,14 @@ aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE ``` -6. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created. -7. If using SSL, create a local private key and certificate, such as with the commands below. Afterwards, set the `KEY` and `CERT` fields in the `.env` file to these, respectively. +8. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created. +9. If using SSL, create a local private key and certificate, such as with the commands below. Afterwards, set the `KEY` and `CERT` fields in the `.env` file to these, respectively. ``` openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048 openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 ``` -8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`. -9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV= tests/run.sh `. To print options, run `tests/run.sh -h`. To run all tests, run `VERSITYGW_TEST_ENV= tests/run_all.sh`. +10. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`. +11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV= tests/run.sh `. To print options, run `tests/run.sh -h`. To run all tests, run `VERSITYGW_TEST_ENV= tests/run_all.sh`. ### Static Bucket Mode diff --git a/tests/commands/put_bucket_acl.sh b/tests/commands/put_bucket_acl.sh index 25e2ee15..122b7c1d 100644 --- a/tests/commands/put_bucket_acl.sh +++ b/tests/commands/put_bucket_acl.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +source ./tests/util_file.sh + put_bucket_acl_s3api() { record_command "put-bucket-acl" "client:$1" if [[ $# -ne 3 ]]; then @@ -14,6 +16,37 @@ put_bucket_acl_s3api() { return 0 } +reset_bucket_acl() { + #if [[ $# -ne 1 ]]; then + # log 2 "'reset_bucket_acl' requires bucket name" + # return 1 + #fi + assert [ $# -eq 1 ] + acl_file="acl_file" + run create_test_files "$acl_file" + assert_success "error creating file" + # shellcheck disable=SC2154 + cat < "$test_file_folder/$acl_file" +{ + "Grants": [ + { + "Grantee": { + "ID": "$AWS_ACCESS_KEY_ID", + "Type": "CanonicalUser" + }, + "Permission": "FULL_CONTROL" + } + ], + "Owner": { + "ID": "$AWS_ACCESS_KEY_ID" + } +} +EOF + run put_bucket_acl_s3api "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$acl_file" + assert_success "error putting bucket ACL" + delete_test_files "$acl_file" +} + put_bucket_canned_acl_s3cmd() { record_command "put-bucket-acl" "client:s3cmd" if [[ $# -ne 2 ]]; then diff --git a/tests/env.sh b/tests/env.sh index 1e91bcc1..9a5d09b7 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -1,169 +1,125 @@ -#!/usr/bin/env bash +#!/usr/bin/env bats + +load ./bats-support/load +load ./bats-assert/load check_env_vars() { - if ! check_universal_vars; then - log 2 "error checking universal params" - return 1 - fi + check_universal_vars + #if ! check_universal_vars; then + # log 2 "error checking universal params" + # return 1 + #fi if [[ $RUN_VERSITYGW == "true" ]]; then - if ! check_versity_vars; then - log 2 "error checking versity params" - return 1 - fi + check_versity_vars fi if [[ $RUN_S3CMD == "true" ]]; then - if [[ -z "$S3CMD_CONFIG" ]]; then - log 2 "running s3cmd commands requires S3CMD_CONFIG param" - return 1 - fi + assert [ -n "$S3CMD_CONFIG" ] export S3CMD_CONFIG fi if [[ $RUN_MC == "true" ]]; then - if [ -z "$MC_ALIAS" ]; then - log 2 "running mc tests requires MC_ALIAS param" - return 1 - fi + assert [ -n "$MC_ALIAS" ] export MC_ALIAS fi return 0 } +source_config_file() { + if [ -z "$VERSITYGW_TEST_ENV" ]; then + if [ -r tests/.env ]; then + source tests/.env + else + log 3 "Warning: no .env file found in tests folder" + fi + else + # shellcheck source=./tests/.env.default + source "$VERSITYGW_TEST_ENV" + fi +} + check_universal_vars() { if [[ $BYPASS_ENV_FILE != "true" ]]; then - if [ -z "$VERSITYGW_TEST_ENV" ]; then - if [ -r tests/.env ]; then - source tests/.env - else - log 3 "Warning: no .env file found in tests folder" - fi - else - # shellcheck source=./tests/.env.default - source "$VERSITYGW_TEST_ENV" - fi + source_config_file fi + if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then # shellcheck source=./tests/.secrets source "$SECRETS_FILE" else log 3 "Warning: no secrets file found" fi + if [[ -n "$LOG_LEVEL" ]]; then export LOG_LEVEL_INT=$LOG_LEVEL fi - if [ -z "$AWS_ACCESS_KEY_ID" ]; then - log 2 "No AWS access key set" - return 1 - elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then - log 2 "No AWS secret access key set" - return 1 - elif [ -z "$AWS_REGION" ]; then - log 2 "No AWS region set" - return 1 - elif [ -z "$AWS_PROFILE" ]; then - log 2 "No AWS profile set" - return 1 - elif [ "$DIRECT" != "true" ] && [ -z "$AWS_ENDPOINT_URL" ]; then - log 2 "No AWS endpoint URL set" - return 1 - elif [[ $RUN_VERSITYGW != "true" ]] && [[ $RUN_VERSITYGW != "false" ]]; then - log 2 "RUN_VERSITYGW must be 'true' or 'false'" - return 1 - elif [ -z "$BUCKET_ONE_NAME" ]; then - log 2 "No bucket one name set" - return 1 - elif [ -z "$BUCKET_TWO_NAME" ]; then - log 2 "No bucket two name set" - return 1 - elif [ -z "$RECREATE_BUCKETS" ]; then - log 2 "No recreate buckets parameter set" - return 1 - elif [[ $RECREATE_BUCKETS != "true" ]] && [[ $RECREATE_BUCKETS != "false" ]]; then - log 2 "RECREATE_BUCKETS must be 'true' or 'false'" - return 1 + + assert [ -n "$AWS_ACCESS_KEY_ID" ] + assert [ -n "$AWS_SECRET_ACCESS_KEY" ] + assert [ -n "$AWS_REGION" ] + assert [ -n "$AWS_PROFILE" ] + if [ "$DIRECT" != "true" ]; then + assert [ -n "$AWS_ENDPOINT_URL" ] fi - export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL RUN_VERSITYGW \ - BUCKET_ONE_NAME BUCKET_TWO_NAME RECREATE_BUCKETS - if [[ -n "$TEST_LOG_FILE" ]]; then - export TEST_LOG_FILE + if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" == "false" ]; then + fail "RUN_VERSITYGW must be 'true' or 'false'" fi - if [[ -n "$VERSITY_LOG_FILE" ]]; then - export VERSITY_LOG_FILE - fi - if [[ -n "$DIRECT" ]]; then - export DIRECT - fi - if [[ -n "$DIRECT_DISPLAY_NAME" ]]; then - export DIRECT_DISPLAY_NAME - fi - if [[ -n "$COVERAGE_DB" ]]; then - export COVERAGE_DB + + assert [ -n "$BUCKET_ONE_NAME" ] + assert [ -n "$BUCKET_TWO_NAME" ] + assert [ -n "$RECREATE_BUCKETS" ] + if [ "$RECREATE_BUCKETS" != "true" ] && [ "$RECREATE_BUCKETS" != "false" ]; then + fail "RECREATE_BUCKETS must be 'true' or 'false'" fi + # exporting these since they're needed for subshells + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL } check_versity_vars() { - if [ -z "$LOCAL_FOLDER" ]; then - log 2 "No local storage folder set" - return 1 - elif [ -z "$VERSITY_EXE" ]; then - log 2 "No versity executable location set" - return 1 - elif [ -z "$BACKEND" ]; then - log 2 "No backend parameter set (options: 'posix', 's3')" - return 1 - fi + assert [ -n "$LOCAL_FOLDER" ] + assert [ -n "$VERSITY_EXE" ] + assert [ -n "$BACKEND" ] export LOCAL_FOLDER VERSITY_EXE BACKEND + if [ "$BACKEND" == 's3' ]; then - if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then - log 2 "missing second AWS access key ID for s3 backend" - return 1 - fi - if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then - log 2 "missing second AWS secret access key for s3 backend" - return 1 - fi + assert [ -n "$AWS_ACCESS_KEY_ID_TWO" ] + assert [ -n "$AWS_SECRET_ACCESS_KEY_TWO" ] export AWS_ACCESS_KEY_ID_TWO AWS_SECRET_ACCESS_KEY_TWO fi + if [[ -r $GOCOVERDIR ]]; then export GOCOVERDIR=$GOCOVERDIR fi + if [[ $RUN_USERS == "true" ]]; then - if ! check_user_vars; then - log 2 "error setting user vars" - return 1 - fi + check_user_vars fi } check_user_vars() { + assert [ -n "$USERNAME_ONE" ] + assert [ -n "$PASSWORD_ONE" ] + assert [ -n "$USERNAME_TWO" ] + assert [ -n "$PASSWORD_TWO" ] + if [[ -z "$IAM_TYPE" ]]; then export IAM_TYPE="folder" fi if [[ "$IAM_TYPE" == "folder" ]]; then - if [[ -z "$USERS_FOLDER" ]]; then - log 2 "if IAM type is folder (or not set), USERS_FOLDER parameter is required" - return 1 - fi + assert [ -n "$USERS_FOLDER" ] if [ ! -d "$USERS_FOLDER" ]; then - if mkdir_error=$(mkdir "$USERS_FOLDER" 2>&1); then - log 2 "error creating users folder: $mkdir_error" - return 1 - fi + mkdir_error=$(mkdir "$USERS_FOLDER" 2>&1) + assert_success "error creating users folder: $mkdir_error" fi IAM_PARAMS="--iam-dir=$USERS_FOLDER" export IAM_PARAMS return 0 fi if [[ $IAM_TYPE == "s3" ]]; then - if [[ -z "$USERS_BUCKET" ]]; then - log 2 "if IAM type is s3, USERS_BUCKET is required" - return 1 - fi + assert [ -n "$USERS_BUCKET" ] IAM_PARAMS="--s3-iam-access $AWS_ACCESS_KEY_ID --s3-iam-secret $AWS_SECRET_ACCESS_KEY \ --s3-iam-region us-east-1 --s3-iam-bucket $USERS_BUCKET --s3-iam-endpoint $AWS_ENDPOINT_URL \ --s3-iam-noverify" export IAM_PARAMS return 0 fi - log 2 "unrecognized IAM_TYPE value: $IAM_TYPE" - return 1 + fail "unrecognized IAM_TYPE value: $IAM_TYPE" } diff --git a/tests/report.sh b/tests/report.sh index a640401c..3f5a5785 100644 --- a/tests/report.sh +++ b/tests/report.sh @@ -72,6 +72,11 @@ $SQL_CREATE_TABLE .exit EOF + RESULT=$(sqlite3 "$COVERAGE_DB" "SELECT name FROM sqlite_master WHERE type='table' AND name='entries';") + if [ -z "$RESULT" ]; then + return + fi + # Iterate over each command in the entries table while IFS="|" read -r command client count; do if [[ $BATS_TEST_STATUS -eq 0 ]]; then diff --git a/tests/setup.sh b/tests/setup.sh index bb812b56..f2b3a15d 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -1,15 +1,13 @@ -#!/usr/bin/env bash +#!/usr/bin/env bats source ./tests/env.sh +source ./tests/report.sh source ./tests/setup_mc.sh source ./tests/versity.sh # bats setup function setup() { - if ! check_env_vars; then - log 2 "error checking env values" - return 1 - fi + check_env_vars if [ "$RUN_VERSITYGW" == "true" ]; then if ! run_versity_app; then log 2 "error starting versity apps" @@ -38,17 +36,15 @@ setup() { fi fi - export AWS_PROFILE \ - BUCKET_ONE_NAME \ - BUCKET_TWO_NAME + export AWS_PROFILE } # fail a test # param: error message -fail() { - log 1 "$1" - return 1 -} +#fail() { +# log 1 "$1" +# exit 1 +#} # bats teardown function teardown() { diff --git a/tests/teardown_static.sh b/tests/teardown_static.sh index bef43808..0705352b 100755 --- a/tests/teardown_static.sh +++ b/tests/teardown_static.sh @@ -1,7 +1,18 @@ #!/bin/bash source ./tests/setup.sh -setup -aws --no-verify-ssl s3 rb s3://"$BUCKET_ONE_NAME" -aws --no-verify-ssl s3 rb s3://"$BUCKET_TWO_NAME" -teardown \ No newline at end of file +source ./tests/util.sh + +if ! setup; then + log 2 "error starting versity to set up static buckets" + exit 1 +fi +if ! delete_bucket_recursive "s3" "$BUCKET_ONE_NAME"; then + log 2 "error creating static bucket one" +elif ! delete_bucket_recursive "s3" "$BUCKET_TWO_NAME"; then + log 2 "error creating static bucket two" +fi +log 4 "buckets deleted successfully" +if ! teardown; then + log 2 "error stopping versity" +fi \ No newline at end of file diff --git a/tests/test_aws.sh b/tests/test_aws.sh index 789456c9..1d6daa4f 100755 --- a/tests/test_aws.sh +++ b/tests/test_aws.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bats +load ./bats-support/load +load ./bats-assert/load + source ./tests/setup.sh source ./tests/util.sh source ./tests/util_aws.sh @@ -351,14 +354,9 @@ EOF } @test "test_policy_get_object_with_user" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi - policy_file="policy_file" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE test_file="test_file" create_test_files "$test_file" "$policy_file" || fail "error creating policy file" @@ -393,16 +391,11 @@ EOF } @test "test_policy_get_object_specific_file" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi - policy_file="policy_file" test_file="test_file" test_file_two="test_file_two" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$policy_file" "$test_file" "$test_file_two" || fail "error creating policy file" echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_file" @@ -432,16 +425,11 @@ EOF } @test "test_policy_get_object_file_wildcard" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi - policy_file="policy_file_one" policy_file_two="policy_file_two" policy_file_three="policy_fil" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$policy_file" "$policy_file_two" "$policy_file_three" || fail "error creating policy file" echo "$BATS_TEST_NAME" >> "$test_file_folder/$policy_file" @@ -471,16 +459,11 @@ EOF } @test "test_policy_get_object_folder_wildcard" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi - policy_file="policy_file" test_folder="test_folder" test_file="test_file" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_folder "$test_folder" || fail "error creating test folder" create_test_files "$test_folder/$test_file" "$policy_file" || fail "error creating policy file, test file" @@ -507,8 +490,8 @@ EOF @test "test_policy_allow_deny" { policy_file="policy_file" test_file="test_file" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$policy_file" "$test_file" || fail "error creating policy file" @@ -550,16 +533,11 @@ EOF } @test "test_policy_deny" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi - policy_file="policy_file" test_file_one="test_file_one" test_file_two="test_file_two" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$test_file_one" "$test_file_two" "$policy_file" || fail "error creating policy file, test file" @@ -599,16 +577,11 @@ EOF } @test "test_policy_put_wildcard" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi - policy_file="policy_file" test_folder="test_folder" test_file="test_file" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_folder "$test_folder" || fail "error creating test folder" create_test_files "$test_folder/$test_file" "$policy_file" || fail "error creating policy file, test file" @@ -640,15 +613,11 @@ EOF } @test "test_policy_delete" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi policy_file="policy_file" test_file_one="test_file_one" test_file_two="test_file_two" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$test_file_one" "$test_file_two" "$policy_file" || fail "error creating policy file, test files" echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_file_one" @@ -678,13 +647,9 @@ EOF } @test "test_policy_get_bucket_policy" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi policy_file="policy_file" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$policy_file" || fail "error creating policy file, test files" @@ -713,14 +678,10 @@ EOF } @test "test_policy_list_multipart_uploads" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi policy_file="policy_file" test_file="test_file" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$policy_file" || fail "error creating policy file, test files" create_large_file "$test_file" || error creating file "$test_file" @@ -729,14 +690,19 @@ EOF principal="$username" action="s3:ListBucketMultipartUploads" resource="arn:aws:s3:::$BUCKET_ONE_NAME" - setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" - setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy" - create_multipart_upload "$BUCKET_ONE_NAME" "$test_file" || fail "error creating multipart upload" + get_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "error getting bucket policy" + log 5 "BUCKET POLICY: $bucket_policy" + get_bucket_acl "s3api" "$BUCKET_ONE_NAME" || fail "error getting bucket ACL" + log 5 "ACL: $acl" + run setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" + assert_success "failed to set up policy" + run create_multipart_upload "$BUCKET_ONE_NAME" "$test_file" + assert_success "failed to create multipart upload" if list_multipart_uploads_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then - log 2 "able to list multipart uploads despite lack of permissions" + fail "able to list multipart uploads despite lack of permissions" fi # shellcheck disable=SC2154 [[ "$list_multipart_uploads_error" == *"Access Denied"* ]] || fail "invalid list multipart uploads error: $list_multipart_uploads_error" @@ -750,14 +716,10 @@ EOF } @test "test_policy_put_bucket_policy" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi policy_file="policy_file" policy_file_two="policy_file_two" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$policy_file" || fail "error creating policy file, test files" @@ -788,13 +750,9 @@ EOF } @test "test_policy_delete_bucket_policy" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi policy_file="policy_file" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$policy_file" || fail "error creating policy file, test files" @@ -817,13 +775,9 @@ EOF } @test "test_policy_get_bucket_acl" { - # TODO (https://github.com/versity/versitygw/issues/637) - if [[ $RECREATE_BUCKETS == "false" ]]; then - return 0 - fi policy_file="policy_file" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE create_test_files "$policy_file" || fail "error creating policy file, test files" @@ -835,7 +789,6 @@ EOF setup_user "$username" "$password" "user" || fail "error creating user" setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket" - #put_bucket_canned_acl "$BUCKET_ONE_NAME" "private" || fail "error putting bucket canned ACL" if get_bucket_acl_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then fail "user able to get bucket ACLs despite permissions" fi @@ -922,7 +875,7 @@ EOF @test "test_policy_abort_multipart_upload" { policy_file="policy_file" test_file="test_file" - username="ABCDEFG" + username=$USERNAME_ONE create_test_files "$policy_file" || fail "error creating policy file" create_large_file "$test_file" || fail "error creating large file" @@ -935,7 +888,7 @@ EOF # shellcheck disable=SC2154 password=$secret_key else - password="HIJLKMN" + password=$PASSWORD_ONE setup_user "$username" "$password" "user" || fail "error setting up user $username" principal="\"$username\"" fi @@ -989,16 +942,60 @@ EOF delete_test_files "$policy_file" "$test_file" } +@test "test_aws_policy_two_principals" { + policy_file="policy_file" + test_file="test_file" + + create_test_files "$test_file" "$policy_file" + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success "error setting up bucket $BUCKET_ONE_NAME" + run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user" + assert_success "error setting up user $USERNAME_ONE" + run setup_user "$USERNAME_TWO" "$PASSWORD_TWO" "user" + assert_success "error setting up user $USERNAME_TWO" + + run put_object "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success "error adding object to bucket" + run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE" + assert_failure "able to get object with user $USERNAME_ONE despite lack of permission" + + run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO" + assert_failure "able to get object with user $USERNAME_TWO despite lack of permission" + + cat < "$test_file_folder"/$policy_file +{ + "Statement": [ + { + "Effect": "Allow", + "Principal": ["$USERNAME_ONE","$USERNAME_TWO"], + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*" + } + ] +} +EOF + + run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" + assert_success "error putting policy" + run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE" + assert_success "error getting object with user $USERNAME_ONE" + run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO" + assert_success "error getting object with user $USERNAME_TWO" + + delete_test_files "$test_file" "$policy_file" "$test_file_folder/copy_one" "$test_file_folder/copy_two" + delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME" +} + @test "test_policy_put_acl" { - if [[ $DIRECT != "true" ]]; then + if [[ $DIRECT != "true" ]] || [[ $RECREATE_BUCKETS == "false" ]]; then # https://github.com/versity/versitygw/issues/702 + # https://github.com/versity/versitygw/issues/716 skip fi policy_file="policy_file" test_file="test_file" - username="ABCDEFG" - password="HIJLKMN" + username=$USERNAME_ONE create_test_files "$policy_file" || fail "error creating policy file" create_large_file "$test_file" || fail "error creating large file" @@ -1014,7 +1011,7 @@ EOF # shellcheck disable=SC2154 password=$secret_key else - password="HIJLKMN" + password=$PASSWORD_ONE setup_user "$username" "$password" "user" || fail "error setting up user $username" principal="\"$username\"" fi diff --git a/tests/test_aws_root_inner.sh b/tests/test_aws_root_inner.sh index c30f7e61..5ebb94c2 100755 --- a/tests/test_aws_root_inner.sh +++ b/tests/test_aws_root_inner.sh @@ -236,14 +236,14 @@ test_get_object_attributes_aws_root() { } test_get_put_object_legal_hold_aws_root() { - # bucket must be created with lock for legal hold - if [[ $RECREATE_BUCKETS == false ]]; then - return + if [[ $RECREATE_BUCKETS == "false" ]]; then + # https://github.com/versity/versitygw/issues/716 + skip fi bucket_file="bucket_file" - username="ABCDEFG" - password="HIJKLMN" + username=$USERNAME_ONE + password=$PASSWORD_ONE legal_hold_retention_setup "$username" "$password" "$bucket_file" @@ -280,11 +280,11 @@ test_get_put_object_legal_hold_aws_root() { test_get_put_object_retention_aws_root() { bucket_file="bucket_file" - username="ABCDEFG" - secret_key="HIJKLMN" + username=$USERNAME_ONE + secret_key=$PASSWORD_ONE - # TODO remove after able to change bucket owner back to root user if [[ $RECREATE_BUCKETS == "false" ]]; then + # https://github.com/versity/versitygw/issues/716 skip fi @@ -328,9 +328,13 @@ test_get_put_object_retention_aws_root() { } test_retention_bypass_aws_root() { + if [[ $RECREATE_BUCKETS == "false" ]]; then + # https://github.com/versity/versitygw/issues/716 + skip + fi bucket_file="bucket_file" - username="ABCDEFG" - secret_key="HIJKLMN" + username=$USERNAME_ONE + secret_key=$PASSWORD_ONE policy_file="policy_file" legal_hold_retention_setup "$username" "$secret_key" "$bucket_file" diff --git a/tests/test_common.sh b/tests/test_common.sh index 72a3ceee..b7813bc8 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -431,7 +431,7 @@ test_put_bucket_acl_s3cmd() { setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || fail "error creating bucket" put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" || fail "error putting bucket ownership controls" - username="abcdefgh" + username=$USERNAME_ONE if [[ $DIRECT != "true" ]]; then setup_user "$username" "HIJKLMN" "user" || fail "error creating user" fi @@ -472,11 +472,15 @@ test_put_bucket_acl_s3cmd() { } test_common_put_bucket_acl() { + if [[ $RECREATE_BUCKETS == "false" ]]; then + # https://github.com/versity/versitygw/issues/716 + skip + fi [[ $# -eq 1 ]] || fail "test common put bucket acl missing command type" setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error creating bucket" put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" || fail "error putting bucket ownership controls" - username="ABCDEFG" + username=$USERNAME_ONE setup_user "$username" "HIJKLMN" "user" || fail "error creating user" get_bucket_acl "$1" "$BUCKET_ONE_NAME" || fail "error retrieving acl" diff --git a/tests/test_user_aws.sh b/tests/test_user_aws.sh index d3f56baa..2a215c85 100755 --- a/tests/test_user_aws.sh +++ b/tests/test_user_aws.sh @@ -30,8 +30,8 @@ export RUN_USERS=true } @test "test_user_get_object" { - username="ABCDEFG" - password="HIJKLMN" + username="$USERNAME_ONE" + password="$USERNAME_ONE" test_file="test_file" setup_user "$username" "$password" "user" || fail "error creating user if nonexistent" @@ -47,8 +47,8 @@ export RUN_USERS=true } @test "test_userplus_get_object" { - username="ABCDEFG" - password="HIJKLMN" + username="$USERNAME_ONE" + password="$PASSWORD_ONE" test_file="test_file" setup_user "$username" "$password" "admin" || fail "error creating user if nonexistent" @@ -64,8 +64,8 @@ export RUN_USERS=true } @test "test_user_delete_object" { - username="ABCDEFG" - password="HIJKLMN" + username="$USERNAME_ONE" + password="$PASSWORD_ONE" test_file="test_file" setup_user "$username" "$password" "user" || fail "error creating user if nonexistent" @@ -81,8 +81,8 @@ export RUN_USERS=true } @test "test_admin_put_get_object" { - username="ABCDEFG" - password="HIJKLMN" + username="$USERNAME_ONE" + password="$PASSWORD_ONE" test_file="test_file" setup_user "$username" "$password" "admin" || fail "error creating user if nonexistent" @@ -103,8 +103,8 @@ export RUN_USERS=true } @test "test_user_create_multipart_upload" { - username="ABCDEFG" - password="HIJKLMN" + username="$USERNAME_ONE" + password="$PASSWORD_ONE" test_file="test_file" setup_user "$username" "$password" "user" || fail "error creating user if nonexistent" diff --git a/tests/test_user_common.sh b/tests/test_user_common.sh index 64109353..d572651e 100755 --- a/tests/test_user_common.sh +++ b/tests/test_user_common.sh @@ -11,10 +11,10 @@ test_admin_user() { fail "test admin user command requires command type" fi - admin_username="ABCDEF" - user_username="GHIJKL" - admin_password="123456" - user_password="789012" + admin_username="$USERNAME_ONE" + admin_password="$PASSWORD_ONE" + user_username="$USERNAME_TWO" + user_password="$PASSWORD_TWO" setup_user "$admin_username" "$admin_password" "admin" || fail "error setting up admin user" @@ -56,8 +56,8 @@ test_create_user_already_exists() { fail "test admin user command requires command type" fi - username="ABCDEG" - password="123456" + username="$USERNAME_ONE" + password="$PASSWORD_ONE" setup_user "$username" "123456" "admin" || fail "error setting up user" if create_user "$username" "123456" "admin"; then @@ -73,8 +73,8 @@ test_user_user() { fail "test admin user command requires command type" fi - username="ABCDEG" - password="123456" + username="$USERNAME_ONE" + password="$PASSWORD_ONE" setup_user "$username" "$password" "user" || fail "error setting up user" delete_bucket "aws" "versity-gwtest-user-bucket" @@ -115,8 +115,8 @@ test_userplus_operation() { fail "test admin user command requires command type" fi - username="ABCDEG" - password="123456" + username="$USERNAME_ONE" + password="$PASSWORD_ONE" delete_bucket "aws" "versity-gwtest-userplus-bucket" setup_user "$username" "$password" "userplus" || fail "error creating user '$username'" diff --git a/tests/util.sh b/tests/util.sh index ae35d711..120d649f 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -127,7 +127,8 @@ clear_bucket_s3api() { fi done delete_bucket_policy "s3api" "$1" || fail "error deleting bucket policy" - put_bucket_canned_acl "$1" "private" || fail "error deleting bucket ACLs" + # TODO uncomment after #716 is fixed + #reset_bucket_acl "$1" || fail "error resetting bucket ACLs" put_object_lock_configuration_disabled "$1" || fail "error removing object lock config" #change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID" || fail "error changing bucket owner" } @@ -199,6 +200,34 @@ bucket_exists() { return 0 } +abort_all_multipart_uploads() { + assert [ $# -eq 1 ] + run aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" + # shellcheck disable=SC2154 + assert_success "error listing uploads: $output" + log 5 "UPLOADS: $output" + if ! upload_set=$(echo "$output" | grep -v "InsecureRequestWarning" | jq -c '.Uploads[]' 2>&1); then + if [[ $upload_set == *"Cannot iterate over null"* ]]; then + return 0 + fi + fail "error getting upload set: $upload_set" + fi + log 5 "UPLOAD SET: $upload_set" + for upload in $upload_set; do + log 5 "UPLOAD: $upload" + upload_id=$(echo "$upload" | jq -r ".UploadId" 2>&1) + assert [ $? -eq 0 ] + log 5 "upload ID: $upload_id" + key=$(echo "$upload" | jq -r ".Key" 2>&1) + assert [ $? -eq 0 ] + log 5 "Key: $key" + + log 5 "Aborting multipart upload for key: $key, UploadId: $upload_id" + run aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$key" --upload-id "$upload_id" + assert_success "error aborting upload: $output" + done +} + # delete buckets or just the contents depending on RECREATE_BUCKETS parameter # params: command type, bucket name # return: 0 for success, 1 for failure @@ -225,6 +254,8 @@ delete_bucket_or_contents() { log 2 "error resetting bucket ACLs" return 1 fi + run abort_all_multipart_uploads "$2" + assert_success "error aborting multipart uploads" log 5 "bucket contents, policy, ACL deletion success" return 0 fi @@ -267,10 +298,7 @@ delete_bucket_or_contents_if_exists() { # param: bucket name # return 0 for success, 1 for failure setup_bucket() { - if [ $# -ne 2 ]; then - log 2 "bucket creation function requires command type, bucket name" - return 1 - fi + assert [ $# -eq 2 ] if [[ $1 == "s3cmd" ]]; then log 5 "putting bucket ownership controls" put_bucket_ownership_controls "$2" "BucketOwnerPreferred" diff --git a/tests/util_file.sh b/tests/util_file.sh index 7d500ba5..9b328fba 100644 --- a/tests/util_file.sh +++ b/tests/util_file.sh @@ -6,19 +6,19 @@ source ./tests/logger.sh # params: filename # export test file folder on success, return 1 for error create_test_files() { - if [ $# -lt 1 ]; then - echo "create test files command missing filename" - return 1 - fi + assert [ $# -gt 0 ] test_file_folder=$PWD if [[ -z "$GITHUB_ACTIONS" ]]; then create_test_file_folder fi for name in "$@"; do if [[ -e "$test_file_folder/$name" ]]; then - error=$(rm "$test_file_folder/$name" 2>&1) || fail "error removing existing test file: $error" + run rm "$test_file_folder/$name" + # shellcheck disable=SC2154 + assert_success "error removing existing test file: $output" fi - error=$(touch "$test_file_folder"/"$name" 2>&1) || fail "error creating new file: $error" + run touch "$test_file_folder"/"$name" + assert_success "error creating new file: $output" done export test_file_folder } @@ -127,10 +127,9 @@ create_test_file_folder() { test_file_folder=$PWD/versity-gwtest fi if ! error=$(mkdir -p "$test_file_folder" 2>&1); then - if [[ $error != *"File exists"* ]]; then - log 2 "error creating test file folder: $error" - return 1 - fi + # shellcheck disable=SC2035 + run [[ "$error" == *"File exists"* ]] + assert_success "error creating test file folder: $error" fi export test_file_folder } diff --git a/tests/versity.sh b/tests/versity.sh index 658414b5..008f4a4f 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -176,11 +176,14 @@ stop_versity() { if [ "$RUN_VERSITYGW" == "false" ]; then return fi + if [[ -z "$versitygw_pid_1" ]]; then + return + fi # shellcheck disable=SC2154 if ! stop_single_process "$versitygw_pid_1"; then log 2 "error stopping versity process" fi - if [[ $BACKEND == 's3' ]]; then + if [[ $BACKEND == 's3' ]] && [[ -n "$versitygw_pid_2" ]]; then # shellcheck disable=SC2154 if ! stop_single_process "$versitygw_pid_2"; then log 2 "error stopping versity process two"