mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-04-23 14:10:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: multiple policy principals, improved bucket cleanup, general cleanup

Browse Source
This commit is contained in:
Luke McCrone
2024-08-02 20:24:37 -03:00
parent adbc8140ed
commit 0facfdc9fd
16 changed files with 322 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/system.yml vendored
View File

@@ -103,6 +103,8 @@ jobs:
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core && ./install.sh $HOME
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
- name: Install s3cmd
run: |
@@ -135,6 +137,10 @@ jobs:
MC_ALIAS: versity
LOG_LEVEL: 4
GOCOVERDIR: ${{ github.workspace }}/cover
USERNAME_ONE: ABCDEFG
PASSWORD_ONE: 1234567
USERNAME_TWO: HIJKLMN
PASSWORD_TWO: 8901234
run: |
make testbin
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST

9
tests/.env.default
View File

@@ -17,4 +17,11 @@ GOCOVERDIR=$PWD/cover
USERS_FOLDER=$PWD/iam
#TEST_LOG_FILE=test.log
#VERSITY_LOG_FILE=versity.log
IAM_TYPE=folder
IAM_TYPE=folder
DIRECT=false
#DIRECT_DISPLAY_NAME=
#COVERAGE_DB=coverage.sql
USERNAME_ONE=ABCDEFG
PASSWORD_ONE=HIJKLMN
USERNAME_TWO=HIJKLMN
PASSWORD_TWO=OPQRSTU

17
tests/README.md
View File

@@ -9,10 +9,11 @@
* **aws cli**: Instructions are [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
* **s3cmd**: Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md).
* **mc**: Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html).
3. Install BATS. Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
4. If running on Mac OS, install **jq** with the command `brew install jq`.
4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile. Example:
3. Install **BATS**. Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
4. Install **bats-support** and **bats-assert**. This can be done by saving the root folder of each repo (both located in https://github.com/bats-core/) in the `tests` folder.
5. If running on Mac OS, install **jq** with the command `brew install jq`.
6. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
7. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile. Example:
```
export AWS_PROFILE=versity-test
export AWS_ACCESS_KEY_ID=<your account ID>
@@ -22,14 +23,14 @@
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE
aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
```
6. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created.
7. If using SSL, create a local private key and certificate, such as with the commands below. Afterwards, set the `KEY` and `CERT` fields in the `.env` file to these, respectively.
8. Create an environment file (`.env`) similar to the ones in this folder, setting the `AWS_PROFILE` parameter to the name of the profile you created.
9. If using SSL, create a local private key and certificate, such as with the commands below. Afterwards, set the `KEY` and `CERT` fields in the `.env` file to these, respectively.
```
openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
```
8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`. To print options, run `tests/run.sh -h`. To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
10. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`. To print options, run `tests/run.sh -h`. To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
### Static Bucket Mode

33
tests/commands/put_bucket_acl.sh
View File

@@ -1,5 +1,7 @@
#!/usr/bin/env bash
source ./tests/util_file.sh
put_bucket_acl_s3api() {
record_command "put-bucket-acl" "client:$1"
if [[ $# -ne 3 ]]; then
@@ -14,6 +16,37 @@ put_bucket_acl_s3api() {
return 0
}
reset_bucket_acl() {
#if [[ $# -ne 1 ]]; then
# log 2 "'reset_bucket_acl' requires bucket name"
# return 1
#fi
assert [ $# -eq 1 ]
acl_file="acl_file"
run create_test_files "$acl_file"
assert_success "error creating file"
# shellcheck disable=SC2154
cat <<EOF > "$test_file_folder/$acl_file"
{
"Grants": [
{
"Grantee": {
"ID": "$AWS_ACCESS_KEY_ID",
"Type": "CanonicalUser"
},
"Permission": "FULL_CONTROL"
}
],
"Owner": {
"ID": "$AWS_ACCESS_KEY_ID"
}
}
EOF
run put_bucket_acl_s3api "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$acl_file"
assert_success "error putting bucket ACL"
delete_test_files "$acl_file"
}
put_bucket_canned_acl_s3cmd() {
record_command "put-bucket-acl" "client:s3cmd"
if [[ $# -ne 2 ]]; then

172
tests/env.sh
View File

@@ -1,169 +1,125 @@
#!/usr/bin/env bash
#!/usr/bin/env bats
load ./bats-support/load
load ./bats-assert/load
check_env_vars() {
if ! check_universal_vars; then
log 2 "error checking universal params"
return 1
fi
check_universal_vars
#if ! check_universal_vars; then
# log 2 "error checking universal params"
# return 1
#fi
if [[ $RUN_VERSITYGW == "true" ]]; then
if ! check_versity_vars; then
log 2 "error checking versity params"
return 1
fi
check_versity_vars
fi
if [[ $RUN_S3CMD == "true" ]]; then
if [[ -z "$S3CMD_CONFIG" ]]; then
log 2 "running s3cmd commands requires S3CMD_CONFIG param"
return 1
fi
assert [ -n "$S3CMD_CONFIG" ]
export S3CMD_CONFIG
fi
if [[ $RUN_MC == "true" ]]; then
if [ -z "$MC_ALIAS" ]; then
log 2 "running mc tests requires MC_ALIAS param"
return 1
fi
assert [ -n "$MC_ALIAS" ]
export MC_ALIAS
fi
return 0
}
source_config_file() {
if [ -z "$VERSITYGW_TEST_ENV" ]; then
if [ -r tests/.env ]; then
source tests/.env
else
log 3 "Warning: no .env file found in tests folder"
fi
else
# shellcheck source=./tests/.env.default
source "$VERSITYGW_TEST_ENV"
fi
}
check_universal_vars() {
if [[ $BYPASS_ENV_FILE != "true" ]]; then
if [ -z "$VERSITYGW_TEST_ENV" ]; then
if [ -r tests/.env ]; then
source tests/.env
else
log 3 "Warning: no .env file found in tests folder"
fi
else
# shellcheck source=./tests/.env.default
source "$VERSITYGW_TEST_ENV"
fi
source_config_file
fi
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
# shellcheck source=./tests/.secrets
source "$SECRETS_FILE"
else
log 3 "Warning: no secrets file found"
fi
if [[ -n "$LOG_LEVEL" ]]; then
export LOG_LEVEL_INT=$LOG_LEVEL
fi
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
log 2 "No AWS access key set"
return 1
elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
log 2 "No AWS secret access key set"
return 1
elif [ -z "$AWS_REGION" ]; then
log 2 "No AWS region set"
return 1
elif [ -z "$AWS_PROFILE" ]; then
log 2 "No AWS profile set"
return 1
elif [ "$DIRECT" != "true" ] && [ -z "$AWS_ENDPOINT_URL" ]; then
log 2 "No AWS endpoint URL set"
return 1
elif [[ $RUN_VERSITYGW != "true" ]] && [[ $RUN_VERSITYGW != "false" ]]; then
log 2 "RUN_VERSITYGW must be 'true' or 'false'"
return 1
elif [ -z "$BUCKET_ONE_NAME" ]; then
log 2 "No bucket one name set"
return 1
elif [ -z "$BUCKET_TWO_NAME" ]; then
log 2 "No bucket two name set"
return 1
elif [ -z "$RECREATE_BUCKETS" ]; then
log 2 "No recreate buckets parameter set"
return 1
elif [[ $RECREATE_BUCKETS != "true" ]] && [[ $RECREATE_BUCKETS != "false" ]]; then
log 2 "RECREATE_BUCKETS must be 'true' or 'false'"
return 1
assert [ -n "$AWS_ACCESS_KEY_ID" ]
assert [ -n "$AWS_SECRET_ACCESS_KEY" ]
assert [ -n "$AWS_REGION" ]
assert [ -n "$AWS_PROFILE" ]
if [ "$DIRECT" != "true" ]; then
assert [ -n "$AWS_ENDPOINT_URL" ]
fi
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL RUN_VERSITYGW \
BUCKET_ONE_NAME BUCKET_TWO_NAME RECREATE_BUCKETS
if [[ -n "$TEST_LOG_FILE" ]]; then
export TEST_LOG_FILE
if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" == "false" ]; then
fail "RUN_VERSITYGW must be 'true' or 'false'"
fi
if [[ -n "$VERSITY_LOG_FILE" ]]; then
export VERSITY_LOG_FILE
fi
if [[ -n "$DIRECT" ]]; then
export DIRECT
fi
if [[ -n "$DIRECT_DISPLAY_NAME" ]]; then
export DIRECT_DISPLAY_NAME
fi
if [[ -n "$COVERAGE_DB" ]]; then
export COVERAGE_DB
assert [ -n "$BUCKET_ONE_NAME" ]
assert [ -n "$BUCKET_TWO_NAME" ]
assert [ -n "$RECREATE_BUCKETS" ]
if [ "$RECREATE_BUCKETS" != "true" ] && [ "$RECREATE_BUCKETS" != "false" ]; then
fail "RECREATE_BUCKETS must be 'true' or 'false'"
fi
# exporting these since they're needed for subshells
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
}
check_versity_vars() {
if [ -z "$LOCAL_FOLDER" ]; then
log 2 "No local storage folder set"
return 1
elif [ -z "$VERSITY_EXE" ]; then
log 2 "No versity executable location set"
return 1
elif [ -z "$BACKEND" ]; then
log 2 "No backend parameter set (options: 'posix', 's3')"
return 1
fi
assert [ -n "$LOCAL_FOLDER" ]
assert [ -n "$VERSITY_EXE" ]
assert [ -n "$BACKEND" ]
export LOCAL_FOLDER VERSITY_EXE BACKEND
if [ "$BACKEND" == 's3' ]; then
if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then
log 2 "missing second AWS access key ID for s3 backend"
return 1
fi
if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then
log 2 "missing second AWS secret access key for s3 backend"
return 1
fi
assert [ -n "$AWS_ACCESS_KEY_ID_TWO" ]
assert [ -n "$AWS_SECRET_ACCESS_KEY_TWO" ]
export AWS_ACCESS_KEY_ID_TWO AWS_SECRET_ACCESS_KEY_TWO
fi
if [[ -r $GOCOVERDIR ]]; then
export GOCOVERDIR=$GOCOVERDIR
fi
if [[ $RUN_USERS == "true" ]]; then
if ! check_user_vars; then
log 2 "error setting user vars"
return 1
fi
check_user_vars
fi
}
check_user_vars() {
assert [ -n "$USERNAME_ONE" ]
assert [ -n "$PASSWORD_ONE" ]
assert [ -n "$USERNAME_TWO" ]
assert [ -n "$PASSWORD_TWO" ]
if [[ -z "$IAM_TYPE" ]]; then
export IAM_TYPE="folder"
fi
if [[ "$IAM_TYPE" == "folder" ]]; then
if [[ -z "$USERS_FOLDER" ]]; then
log 2 "if IAM type is folder (or not set), USERS_FOLDER parameter is required"
return 1
fi
assert [ -n "$USERS_FOLDER" ]
if [ ! -d "$USERS_FOLDER" ]; then
if mkdir_error=$(mkdir "$USERS_FOLDER" 2>&1); then
log 2 "error creating users folder: $mkdir_error"
return 1
fi
mkdir_error=$(mkdir "$USERS_FOLDER" 2>&1)
assert_success "error creating users folder: $mkdir_error"
fi
IAM_PARAMS="--iam-dir=$USERS_FOLDER"
export IAM_PARAMS
return 0
fi
if [[ $IAM_TYPE == "s3" ]]; then
if [[ -z "$USERS_BUCKET" ]]; then
log 2 "if IAM type is s3, USERS_BUCKET is required"
return 1
fi
assert [ -n "$USERS_BUCKET" ]
IAM_PARAMS="--s3-iam-access $AWS_ACCESS_KEY_ID --s3-iam-secret $AWS_SECRET_ACCESS_KEY \
--s3-iam-region us-east-1 --s3-iam-bucket $USERS_BUCKET --s3-iam-endpoint $AWS_ENDPOINT_URL \
--s3-iam-noverify"
export IAM_PARAMS
return 0
fi
log 2 "unrecognized IAM_TYPE value: $IAM_TYPE"
return 1
fail "unrecognized IAM_TYPE value: $IAM_TYPE"
}

5
tests/report.sh
View File

@@ -72,6 +72,11 @@ $SQL_CREATE_TABLE
.exit
EOF
RESULT=$(sqlite3 "$COVERAGE_DB" "SELECT name FROM sqlite_master WHERE type='table' AND name='entries';")
if [ -z "$RESULT" ]; then
return
fi
# Iterate over each command in the entries table
while IFS="|" read -r command client count; do
if [[ $BATS_TEST_STATUS -eq 0 ]]; then

20
tests/setup.sh
View File

@@ -1,15 +1,13 @@
#!/usr/bin/env bash
#!/usr/bin/env bats
source ./tests/env.sh
source ./tests/report.sh
source ./tests/setup_mc.sh
source ./tests/versity.sh
# bats setup function
setup() {
if ! check_env_vars; then
log 2 "error checking env values"
return 1
fi
check_env_vars
if [ "$RUN_VERSITYGW" == "true" ]; then
if ! run_versity_app; then
log 2 "error starting versity apps"
@@ -38,17 +36,15 @@ setup() {
fi
fi
export AWS_PROFILE \
BUCKET_ONE_NAME \
BUCKET_TWO_NAME
export AWS_PROFILE
}
# fail a test
# param: error message
fail() {
log 1 "$1"
return 1
}
#fail() {
# log 1 "$1"
# exit 1
#}
# bats teardown function
teardown() {

19
tests/teardown_static.sh
View File

@@ -1,7 +1,18 @@
#!/bin/bash
source ./tests/setup.sh
setup
aws --no-verify-ssl s3 rb s3://"$BUCKET_ONE_NAME"
aws --no-verify-ssl s3 rb s3://"$BUCKET_TWO_NAME"
teardown
source ./tests/util.sh
if ! setup; then
log 2 "error starting versity to set up static buckets"
exit 1
fi
if ! delete_bucket_recursive "s3" "$BUCKET_ONE_NAME"; then
log 2 "error creating static bucket one"
elif ! delete_bucket_recursive "s3" "$BUCKET_TWO_NAME"; then
log 2 "error creating static bucket two"
fi
log 4 "buckets deleted successfully"
if ! teardown; then
log 2 "error stopping versity"
fi

179
tests/test_aws.sh
View File

@@ -1,5 +1,8 @@
#!/usr/bin/env bats
load ./bats-support/load
load ./bats-assert/load
source ./tests/setup.sh
source ./tests/util.sh
source ./tests/util_aws.sh
@@ -351,14 +354,9 @@ EOF
}
@test "test_policy_get_object_with_user" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
test_file="test_file"
create_test_files "$test_file" "$policy_file" || fail "error creating policy file"
@@ -393,16 +391,11 @@ EOF
}
@test "test_policy_get_object_specific_file" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
test_file="test_file"
test_file_two="test_file_two"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$policy_file" "$test_file" "$test_file_two" || fail "error creating policy file"
echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_file"
@@ -432,16 +425,11 @@ EOF
}
@test "test_policy_get_object_file_wildcard" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file_one"
policy_file_two="policy_file_two"
policy_file_three="policy_fil"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$policy_file" "$policy_file_two" "$policy_file_three" || fail "error creating policy file"
echo "$BATS_TEST_NAME" >> "$test_file_folder/$policy_file"
@@ -471,16 +459,11 @@ EOF
}
@test "test_policy_get_object_folder_wildcard" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
test_folder="test_folder"
test_file="test_file"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_folder "$test_folder" || fail "error creating test folder"
create_test_files "$test_folder/$test_file" "$policy_file" || fail "error creating policy file, test file"
@@ -507,8 +490,8 @@ EOF
@test "test_policy_allow_deny" {
policy_file="policy_file"
test_file="test_file"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$policy_file" "$test_file" || fail "error creating policy file"
@@ -550,16 +533,11 @@ EOF
}
@test "test_policy_deny" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
test_file_one="test_file_one"
test_file_two="test_file_two"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$test_file_one" "$test_file_two" "$policy_file" || fail "error creating policy file, test file"
@@ -599,16 +577,11 @@ EOF
}
@test "test_policy_put_wildcard" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
test_folder="test_folder"
test_file="test_file"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_folder "$test_folder" || fail "error creating test folder"
create_test_files "$test_folder/$test_file" "$policy_file" || fail "error creating policy file, test file"
@@ -640,15 +613,11 @@ EOF
}
@test "test_policy_delete" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
test_file_one="test_file_one"
test_file_two="test_file_two"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$test_file_one" "$test_file_two" "$policy_file" || fail "error creating policy file, test files"
echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_file_one"
@@ -678,13 +647,9 @@ EOF
}
@test "test_policy_get_bucket_policy" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$policy_file" || fail "error creating policy file, test files"
@@ -713,14 +678,10 @@ EOF
}
@test "test_policy_list_multipart_uploads" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
test_file="test_file"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$policy_file" || fail "error creating policy file, test files"
create_large_file "$test_file" || error creating file "$test_file"
@@ -729,14 +690,19 @@ EOF
principal="$username"
action="s3:ListBucketMultipartUploads"
resource="arn:aws:s3:::$BUCKET_ONE_NAME"
setup_user "$username" "$password" "user" || fail "error creating user"
setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
create_multipart_upload "$BUCKET_ONE_NAME" "$test_file" || fail "error creating multipart upload"
get_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "error getting bucket policy"
log 5 "BUCKET POLICY: $bucket_policy"
get_bucket_acl "s3api" "$BUCKET_ONE_NAME" || fail "error getting bucket ACL"
log 5 "ACL: $acl"
run setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
assert_success "failed to set up policy"
run create_multipart_upload "$BUCKET_ONE_NAME" "$test_file"
assert_success "failed to create multipart upload"
if list_multipart_uploads_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then
log 2 "able to list multipart uploads despite lack of permissions"
fail "able to list multipart uploads despite lack of permissions"
fi
# shellcheck disable=SC2154
[[ "$list_multipart_uploads_error" == *"Access Denied"* ]] || fail "invalid list multipart uploads error: $list_multipart_uploads_error"
@@ -750,14 +716,10 @@ EOF
}
@test "test_policy_put_bucket_policy" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
policy_file_two="policy_file_two"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$policy_file" || fail "error creating policy file, test files"
@@ -788,13 +750,9 @@ EOF
}
@test "test_policy_delete_bucket_policy" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$policy_file" || fail "error creating policy file, test files"
@@ -817,13 +775,9 @@ EOF
}
@test "test_policy_get_bucket_acl" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
create_test_files "$policy_file" || fail "error creating policy file, test files"
@@ -835,7 +789,6 @@ EOF
setup_user "$username" "$password" "user" || fail "error creating user"
setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
#put_bucket_canned_acl "$BUCKET_ONE_NAME" "private" || fail "error putting bucket canned ACL"
if get_bucket_acl_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then
fail "user able to get bucket ACLs despite permissions"
fi
@@ -922,7 +875,7 @@ EOF
@test "test_policy_abort_multipart_upload" {
policy_file="policy_file"
test_file="test_file"
username="ABCDEFG"
username=$USERNAME_ONE
create_test_files "$policy_file" || fail "error creating policy file"
create_large_file "$test_file" || fail "error creating large file"
@@ -935,7 +888,7 @@ EOF
# shellcheck disable=SC2154
password=$secret_key
else
password="HIJLKMN"
password=$PASSWORD_ONE
setup_user "$username" "$password" "user" || fail "error setting up user $username"
principal="\"$username\""
fi
@@ -989,16 +942,60 @@ EOF
delete_test_files "$policy_file" "$test_file"
}
@test "test_aws_policy_two_principals" {
policy_file="policy_file"
test_file="test_file"
create_test_files "$test_file" "$policy_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success "error setting up bucket $BUCKET_ONE_NAME"
run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user"
assert_success "error setting up user $USERNAME_ONE"
run setup_user "$USERNAME_TWO" "$PASSWORD_TWO" "user"
assert_success "error setting up user $USERNAME_TWO"
run put_object "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success "error adding object to bucket"
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE"
assert_failure "able to get object with user $USERNAME_ONE despite lack of permission"
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO"
assert_failure "able to get object with user $USERNAME_TWO despite lack of permission"
cat <<EOF > "$test_file_folder"/$policy_file
{
"Statement": [
{
"Effect": "Allow",
"Principal": ["$USERNAME_ONE","$USERNAME_TWO"],
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*"
}
]
}
EOF
run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file"
assert_success "error putting policy"
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE"
assert_success "error getting object with user $USERNAME_ONE"
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO"
assert_success "error getting object with user $USERNAME_TWO"
delete_test_files "$test_file" "$policy_file" "$test_file_folder/copy_one" "$test_file_folder/copy_two"
delete_bucket_or_contents "s3api" "$BUCKET_ONE_NAME"
}
@test "test_policy_put_acl" {
if [[ $DIRECT != "true" ]]; then
if [[ $DIRECT != "true" ]] || [[ $RECREATE_BUCKETS == "false" ]]; then
# https://github.com/versity/versitygw/issues/702
# https://github.com/versity/versitygw/issues/716
skip
fi
policy_file="policy_file"
test_file="test_file"
username="ABCDEFG"
password="HIJLKMN"
username=$USERNAME_ONE
create_test_files "$policy_file" || fail "error creating policy file"
create_large_file "$test_file" || fail "error creating large file"
@@ -1014,7 +1011,7 @@ EOF
# shellcheck disable=SC2154
password=$secret_key
else
password="HIJLKMN"
password=$PASSWORD_ONE
setup_user "$username" "$password" "user" || fail "error setting up user $username"
principal="\"$username\""
fi

24
tests/test_aws_root_inner.sh
View File

@@ -236,14 +236,14 @@ test_get_object_attributes_aws_root() {
}
test_get_put_object_legal_hold_aws_root() {
# bucket must be created with lock for legal hold
if [[ $RECREATE_BUCKETS == false ]]; then
return
if [[ $RECREATE_BUCKETS == "false" ]]; then
# https://github.com/versity/versitygw/issues/716
skip
fi
bucket_file="bucket_file"
username="ABCDEFG"
password="HIJKLMN"
username=$USERNAME_ONE
password=$PASSWORD_ONE
legal_hold_retention_setup "$username" "$password" "$bucket_file"
@@ -280,11 +280,11 @@ test_get_put_object_legal_hold_aws_root() {
test_get_put_object_retention_aws_root() {
bucket_file="bucket_file"
username="ABCDEFG"
secret_key="HIJKLMN"
username=$USERNAME_ONE
secret_key=$PASSWORD_ONE
# TODO remove after able to change bucket owner back to root user
if [[ $RECREATE_BUCKETS == "false" ]]; then
# https://github.com/versity/versitygw/issues/716
skip
fi
@@ -328,9 +328,13 @@ test_get_put_object_retention_aws_root() {
}
test_retention_bypass_aws_root() {
if [[ $RECREATE_BUCKETS == "false" ]]; then
# https://github.com/versity/versitygw/issues/716
skip
fi
bucket_file="bucket_file"
username="ABCDEFG"
secret_key="HIJKLMN"
username=$USERNAME_ONE
secret_key=$PASSWORD_ONE
policy_file="policy_file"
legal_hold_retention_setup "$username" "$secret_key" "$bucket_file"

8
tests/test_common.sh
View File

@@ -431,7 +431,7 @@ test_put_bucket_acl_s3cmd() {
setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || fail "error creating bucket"
put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" || fail "error putting bucket ownership controls"
username="abcdefgh"
username=$USERNAME_ONE
if [[ $DIRECT != "true" ]]; then
setup_user "$username" "HIJKLMN" "user" || fail "error creating user"
fi
@@ -472,11 +472,15 @@ test_put_bucket_acl_s3cmd() {
}
test_common_put_bucket_acl() {
if [[ $RECREATE_BUCKETS == "false" ]]; then
# https://github.com/versity/versitygw/issues/716
skip
fi
[[ $# -eq 1 ]] || fail "test common put bucket acl missing command type"
setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error creating bucket"
put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" || fail "error putting bucket ownership controls"
username="ABCDEFG"
username=$USERNAME_ONE
setup_user "$username" "HIJKLMN" "user" || fail "error creating user"
get_bucket_acl "$1" "$BUCKET_ONE_NAME" || fail "error retrieving acl"

20
tests/test_user_aws.sh
View File

@@ -30,8 +30,8 @@ export RUN_USERS=true
}
@test "test_user_get_object" {
username="ABCDEFG"
password="HIJKLMN"
username="$USERNAME_ONE"
password="$USERNAME_ONE"
test_file="test_file"
setup_user "$username" "$password" "user" || fail "error creating user if nonexistent"
@@ -47,8 +47,8 @@ export RUN_USERS=true
}
@test "test_userplus_get_object" {
username="ABCDEFG"
password="HIJKLMN"
username="$USERNAME_ONE"
password="$PASSWORD_ONE"
test_file="test_file"
setup_user "$username" "$password" "admin" || fail "error creating user if nonexistent"
@@ -64,8 +64,8 @@ export RUN_USERS=true
}
@test "test_user_delete_object" {
username="ABCDEFG"
password="HIJKLMN"
username="$USERNAME_ONE"
password="$PASSWORD_ONE"
test_file="test_file"
setup_user "$username" "$password" "user" || fail "error creating user if nonexistent"
@@ -81,8 +81,8 @@ export RUN_USERS=true
}
@test "test_admin_put_get_object" {
username="ABCDEFG"
password="HIJKLMN"
username="$USERNAME_ONE"
password="$PASSWORD_ONE"
test_file="test_file"
setup_user "$username" "$password" "admin" || fail "error creating user if nonexistent"
@@ -103,8 +103,8 @@ export RUN_USERS=true
}
@test "test_user_create_multipart_upload" {
username="ABCDEFG"
password="HIJKLMN"
username="$USERNAME_ONE"
password="$PASSWORD_ONE"
test_file="test_file"
setup_user "$username" "$password" "user" || fail "error creating user if nonexistent"

20
tests/test_user_common.sh
View File

@@ -11,10 +11,10 @@ test_admin_user() {
fail "test admin user command requires command type"
fi
admin_username="ABCDEF"
user_username="GHIJKL"
admin_password="123456"
user_password="789012"
admin_username="$USERNAME_ONE"
admin_password="$PASSWORD_ONE"
user_username="$USERNAME_TWO"
user_password="$PASSWORD_TWO"
setup_user "$admin_username" "$admin_password" "admin" || fail "error setting up admin user"
@@ -56,8 +56,8 @@ test_create_user_already_exists() {
fail "test admin user command requires command type"
fi
username="ABCDEG"
password="123456"
username="$USERNAME_ONE"
password="$PASSWORD_ONE"
setup_user "$username" "123456" "admin" || fail "error setting up user"
if create_user "$username" "123456" "admin"; then
@@ -73,8 +73,8 @@ test_user_user() {
fail "test admin user command requires command type"
fi
username="ABCDEG"
password="123456"
username="$USERNAME_ONE"
password="$PASSWORD_ONE"
setup_user "$username" "$password" "user" || fail "error setting up user"
delete_bucket "aws" "versity-gwtest-user-bucket"
@@ -115,8 +115,8 @@ test_userplus_operation() {
fail "test admin user command requires command type"
fi
username="ABCDEG"
password="123456"
username="$USERNAME_ONE"
password="$PASSWORD_ONE"
delete_bucket "aws" "versity-gwtest-userplus-bucket"
setup_user "$username" "$password" "userplus" || fail "error creating user '$username'"

38
tests/util.sh
View File

@@ -127,7 +127,8 @@ clear_bucket_s3api() {
fi
done
delete_bucket_policy "s3api" "$1" || fail "error deleting bucket policy"
put_bucket_canned_acl "$1" "private" || fail "error deleting bucket ACLs"
# TODO uncomment after #716 is fixed
#reset_bucket_acl "$1" || fail "error resetting bucket ACLs"
put_object_lock_configuration_disabled "$1" || fail "error removing object lock config"
#change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID" || fail "error changing bucket owner"
}
@@ -199,6 +200,34 @@ bucket_exists() {
return 0
}
abort_all_multipart_uploads() {
assert [ $# -eq 1 ]
run aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1"
# shellcheck disable=SC2154
assert_success "error listing uploads: $output"
log 5 "UPLOADS: $output"
if ! upload_set=$(echo "$output" | grep -v "InsecureRequestWarning" | jq -c '.Uploads[]' 2>&1); then
if [[ $upload_set == *"Cannot iterate over null"* ]]; then
return 0
fi
fail "error getting upload set: $upload_set"
fi
log 5 "UPLOAD SET: $upload_set"
for upload in $upload_set; do
log 5 "UPLOAD: $upload"
upload_id=$(echo "$upload" | jq -r ".UploadId" 2>&1)
assert [ $? -eq 0 ]
log 5 "upload ID: $upload_id"
key=$(echo "$upload" | jq -r ".Key" 2>&1)
assert [ $? -eq 0 ]
log 5 "Key: $key"
log 5 "Aborting multipart upload for key: $key, UploadId: $upload_id"
run aws --no-verify-ssl s3api abort-multipart-upload --bucket "$1" --key "$key" --upload-id "$upload_id"
assert_success "error aborting upload: $output"
done
}
# delete buckets or just the contents depending on RECREATE_BUCKETS parameter
# params: command type, bucket name
# return: 0 for success, 1 for failure
@@ -225,6 +254,8 @@ delete_bucket_or_contents() {
log 2 "error resetting bucket ACLs"
return 1
fi
run abort_all_multipart_uploads "$2"
assert_success "error aborting multipart uploads"
log 5 "bucket contents, policy, ACL deletion success"
return 0
fi
@@ -267,10 +298,7 @@ delete_bucket_or_contents_if_exists() {
# param: bucket name
# return 0 for success, 1 for failure
setup_bucket() {
if [ $# -ne 2 ]; then
log 2 "bucket creation function requires command type, bucket name"
return 1
fi
assert [ $# -eq 2 ]
if [[ $1 == "s3cmd" ]]; then
log 5 "putting bucket ownership controls"
put_bucket_ownership_controls "$2" "BucketOwnerPreferred"

19
tests/util_file.sh
View File

@@ -6,19 +6,19 @@ source ./tests/logger.sh
# params: filename
# export test file folder on success, return 1 for error
create_test_files() {
if [ $# -lt 1 ]; then
echo "create test files command missing filename"
return 1
fi
assert [ $# -gt 0 ]
test_file_folder=$PWD
if [[ -z "$GITHUB_ACTIONS" ]]; then
create_test_file_folder
fi
for name in "$@"; do
if [[ -e "$test_file_folder/$name" ]]; then
error=$(rm "$test_file_folder/$name" 2>&1) || fail "error removing existing test file: $error"
run rm "$test_file_folder/$name"
# shellcheck disable=SC2154
assert_success "error removing existing test file: $output"
fi
error=$(touch "$test_file_folder"/"$name" 2>&1) || fail "error creating new file: $error"
run touch "$test_file_folder"/"$name"
assert_success "error creating new file: $output"
done
export test_file_folder
}
@@ -127,10 +127,9 @@ create_test_file_folder() {
test_file_folder=$PWD/versity-gwtest
fi
if ! error=$(mkdir -p "$test_file_folder" 2>&1); then
if [[ $error != *"File exists"* ]]; then
log 2 "error creating test file folder: $error"
return 1
fi
# shellcheck disable=SC2035
run [[ "$error" == *"File exists"* ]]
assert_success "error creating test file folder: $error"
fi
export test_file_folder
}

5
tests/versity.sh
View File

@@ -176,11 +176,14 @@ stop_versity() {
if [ "$RUN_VERSITYGW" == "false" ]; then
return
fi
if [[ -z "$versitygw_pid_1" ]]; then
return
fi
# shellcheck disable=SC2154
if ! stop_single_process "$versitygw_pid_1"; then
log 2 "error stopping versity process"
fi
if [[ $BACKEND == 's3' ]]; then
if [[ $BACKEND == 's3' ]] && [[ -n "$versitygw_pid_2" ]]; then
# shellcheck disable=SC2154
if ! stop_single_process "$versitygw_pid_2"; then
log 2 "error stopping versity process two"