Merge pull request #1219 from versity/test/user_static_autocreation

Test/user static autocreation
This commit is contained in:
Ben McClelland
2025-04-18 09:26:20 -07:00
committed by GitHub
9 changed files with 186 additions and 85 deletions

View File

@@ -173,8 +173,8 @@ jobs:
COMMAND_LOG: command.log
TIME_LOG: time.log
PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
AUTOCREATE_USERS: true
USER_AUTOCREATION_PREFIX: github-actions-test-
AUTOGENERATE_USERS: true
USER_AUTOGENERATION_PREFIX: github-actions-test-
run: |
make testbin
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST

View File

@@ -92,7 +92,7 @@ delete_object_with_user() {
if [[ $1 == 's3' ]]; then
delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3 rm "s3://$2/$3" 2>&1) || exit_code=$?
elif [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" --bypass-governance-retention 2>&1) || exit_code=$?
delete_object_error=$(AWS_ACCESS_KEY_ID="$4" AWS_SECRET_ACCESS_KEY="$5" send_command aws --no-verify-ssl s3api delete-object --bucket "$2" --key "$3" 2>&1) || exit_code=$?
elif [[ $1 == 's3cmd' ]]; then
delete_object_error=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm --access_key="$4" --secret_key="$5" "s3://$2/$3" 2>&1) || exit_code=$?
else

View File

@@ -20,6 +20,41 @@ source ./tests/setup_mc.sh
source ./tests/util/util_object.sh
source ./tests/versity.sh
check_secrets_line() {
if [[ $secrets_line =~ ^(USER_ID_(ADMIN|USERPLUS|USER)_[0-9])= ]]; then
match=${BASH_REMATCH[1]}
role=$(echo -n "${BASH_REMATCH[2]}" | tr '[:upper:]' '[:lower:]')
if [ -z "${!match}" ]; then
log 2 "$match secrets parameter missing"
return 1
fi
username_env="${match/USER_ID/USERNAME}"
password_env="${match/USER_ID/PASSWORD}"
if [ -z "${!username_env}" ]; then
log 2 "$username_env secrets parameter missing"
return 1
fi
if [ -z "${!password_env}" ]; then
log 2 "$password_env secrets parameter missing"
return 1
fi
if ! user_exists "${!username_env}" && ! create_user_versitygw "${!username_env}" "${!password_env}" "$role"; then
log 2 "error creating user"
return 1
fi
fi
return 0
}
static_user_versitygw_setup() {
while read -r secrets_line || [ -n "$secrets_line" ]; do
if ! check_secrets_line; then
return 1
fi
done < "$SECRETS_FILE"
}
# bats setup function
setup() {
base_setup
@@ -32,6 +67,13 @@ setup() {
export TEST_LOG_FILE
fi
if [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ]; then
if ! static_user_versitygw_setup; then
log 2 "error setting up static versitygw users"
return 1
fi
fi
log 4 "Running test $BATS_TEST_NAME"
if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
start_time=$(date +%s)
@@ -83,7 +125,7 @@ teardown() {
if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then
log 3 "error deleting user $USERNAME_TWO"
fi
if [ "$AUTOCREATE_USERS" == "true" ] && ! delete_autocreated_users; then
if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then
log 3 "error deleting autocreated users"
fi
if [ "$REMOVE_TEST_FILE_FOLDER" == "true" ]; then

View File

@@ -270,3 +270,20 @@ export RUN_USERS=true
assert_output -p "Directory object contains data payload"
}
@test "s3api - --bypass-governance-retention w/o bucket w/object lock fails" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1218"
fi
test_file="test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success
run delete_object_bypass_retention "$BUCKET_ONE_NAME" "$test_file" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"
assert_failure
assert_output -p "InvalidArgument"
assert_output -p "x-amz-bypass-governance-retention is only applicable"
}

View File

@@ -20,21 +20,23 @@ source ./tests/util/util_setup.sh
test_s3api_policy_allow_deny() {
policy_file="policy_file"
test_file="test_file"
username=$USERNAME_ONE
password=$PASSWORD_ONE
run create_test_files "$policy_file"
assert_success
run setup_user "$username" "$password" "user"
run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME"
assert_success
# shellcheck disable=SC2154
user_id=${lines[0]}
username=${lines[1]}
password=${lines[2]}
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" \
"Deny" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file" \
"Allow" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
run setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" \
"Deny" "$user_id" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file" \
"Allow" "$user_id" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
assert_success
run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
@@ -51,24 +53,25 @@ test_s3api_policy_delete() {
policy_file="policy_file"
test_file_one="test_file_one"
test_file_two="test_file_two"
username=$USERNAME_ONE
password=$PASSWORD_ONE
run create_test_files "$policy_file"
assert_success
run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME"
assert_success
user_id=${lines[0]}
username=${lines[1]}
password=${lines[2]}
effect="Allow"
principal="$username"
principal="$user_id"
action="s3:DeleteObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file_two"
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$test_file_one" "$test_file_two"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
assert_success
log 5 "Policy: $(cat "$TEST_FILE_FOLDER/$policy_file")"
@@ -92,21 +95,22 @@ test_s3api_policy_deny() {
policy_file="policy_file"
test_file_one="test_file_one"
test_file_two="test_file_two"
username=$USERNAME_ONE
password=$PASSWORD_ONE
run create_test_files "$policy_file"
assert_success
run setup_user "$username" "$password" "user"
run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME"
assert_success
user_id=${lines[0]}
username=${lines[1]}
password=${lines[2]}
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$test_file_one" "$test_file_two"
assert_success
run setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" \
"Deny" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file_two" \
"Allow" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/*"
run setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" \
"Deny" "$user_id" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file_two" \
"Allow" "$user_id" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/*"
assert_success
log 5 "Policy: $(cat "$TEST_FILE_FOLDER/$policy_file")"
@@ -129,21 +133,22 @@ test_s3api_policy_get_object_file_wildcard() {
policy_file="policy_file_one"
policy_file_two="policy_file_two"
policy_file_three="policy_fil"
username=$USERNAME_ONE
password=$PASSWORD_ONE
effect="Allow"
principal="$username"
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/policy_file*"
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$policy_file_two" "$policy_file_three"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME"
assert_success
user_id=${lines[0]}
username=${lines[1]}
password=${lines[2]}
effect="Allow"
principal="$user_id"
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/policy_file*"
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
assert_success
run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
assert_success
@@ -169,24 +174,25 @@ test_s3api_policy_get_object_folder_wildcard() {
policy_file="policy_file"
test_folder="test_folder"
test_file="test_file"
username=$USERNAME_ONE
password=$PASSWORD_ONE
run create_test_folder "$test_folder"
assert_success
effect="Allow"
principal="$username"
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*"
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_folder/$test_file"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME"
assert_success
user_id=${lines[0]}
username=${lines[1]}
password=${lines[2]}
effect="Allow"
principal="$user_id"
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*"
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
assert_success
run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
@@ -203,21 +209,22 @@ test_s3api_policy_get_object_specific_file() {
policy_file="policy_file"
test_file="test_file"
test_file_two="test_file_two"
username=$USERNAME_ONE
password=$PASSWORD_ONE
run setup_user "$username" "$password" "user"
run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME"
assert_success
user_id=${lines[0]}
username=${lines[1]}
password=${lines[2]}
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$test_file" "$test_file_two"
assert_success
effect="Allow"
principal="$username"
principal="$user_id"
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/test_file"
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
assert_success
run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
@@ -237,9 +244,6 @@ test_s3api_policy_get_object_specific_file() {
}
test_s3api_policy_get_object_with_user() {
username=$USERNAME_ONE
password=$PASSWORD_ONE
test_file="test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
@@ -247,15 +251,18 @@ test_s3api_policy_get_object_with_user() {
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success
run setup_user "$username" "$password" "user"
run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME"
assert_success
user_id=${lines[0]}
username=${lines[1]}
password=${lines[2]}
run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
assert_success
policy_file="policy_file"
effect="Allow"
principal="$username"
principal="$user_id"
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
@@ -277,7 +284,7 @@ test_s3api_policy_invalid_action() {
resource="arn:aws:s3:::$BUCKET_ONE_NAME/*"
# shellcheck disable=SC2154
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
@@ -298,16 +305,17 @@ test_s3api_policy_put_wildcard() {
run create_test_folder "$test_folder"
assert_success
run setup_user_versitygw_or_direct "$USERNAME_ONE" "$PASSWORD_ONE" "user" "$BUCKET_ONE_NAME"
run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME"
assert_success
# shellcheck disable=SC2154
user_id=${lines[0]}
username=${lines[1]}
password=${lines[2]}
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_folder/$test_file"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "Allow" "$username" "s3:PutObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*"
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "Allow" "$user_id" "s3:PutObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*"
assert_success
run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
@@ -316,7 +324,7 @@ test_s3api_policy_put_wildcard() {
run put_object_with_user "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password"
assert_failure
# shellcheck disable=SC2154
assert_output -p "Access Denied"
assert_output -p "AccessDenied"
run put_object_with_user "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$username" "$password"
assert_success
@@ -334,26 +342,33 @@ test_s3api_policy_two_principals() {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user"
run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME"
assert_success
run setup_user "$USERNAME_TWO" "$PASSWORD_TWO" "user"
user_id_one=${lines[0]}
username_one=${lines[1]}
password_one=${lines[2]}
run setup_user_v2 "user" 2 "$BUCKET_ONE_NAME"
assert_success
user_id_two=${lines[0]}
username_two=${lines[1]}
password_two=${lines[2]}
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE"
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_one" "$username_one" "$password_one"
assert_failure
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_two" "$username_two" "$password_two"
assert_failure
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO"
assert_failure
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "Allow" "$USERNAME_ONE,$USERNAME_TWO" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/*"
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "Allow" "$user_id_one,$user_id_two" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/*"
assert_success
run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
assert_success
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_one" "$USERNAME_ONE" "$PASSWORD_ONE"
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_one" "$username_one" "$password_one"
assert_success
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO"
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_two" "$username_two" "$password_two"
assert_success
}

View File

@@ -162,6 +162,23 @@ bucket_exists() {
return 1
}
direct_wait_for_bucket() {
if [ $# -ne 1 ]; then
log 2 "'direct_wait_for_bucket' requires bucket name"
return 1
fi
bucket_verification_start_time=$(date +%s)
while ! bucket_exists "s3api" "$1"; do
bucket_verification_end_time=$(date +%s)
if [ $((bucket_verification_end_time-bucket_verification_start_time)) -ge 60 ]; then
log 2 "bucket existence check timeout"
return 1
fi
sleep 5
done
return 0
}
# params: client, bucket name
# return 0 for success, 1 for error
bucket_cleanup() {
@@ -277,8 +294,8 @@ setup_bucket() {
fi
# bucket creation and resets take longer to propagate in direct mode
if [ "$DIRECT" == "true" ]; then
sleep 15
if [ "$DIRECT" == "true" ] && ! direct_wait_for_bucket "$2"; then
return 1
fi
if [[ $1 == "s3cmd" ]]; then

View File

@@ -10,7 +10,7 @@ block_delete_object_without_permission() {
return 1
fi
# shellcheck disable=SC2154
if [[ "$delete_object_error" != *"Access Denied"* ]]; then
if [[ "$delete_object_error" != *"AccessDenied"* ]]; then
log 2 "invalid delete object error: $delete_object_error"
return 1
fi

View File

@@ -44,9 +44,9 @@ check_for_empty_policy() {
add_direct_user_to_principal() {
if [ "${principals[$idx]}" == "*" ]; then
modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/$DIRECT_S3_ROOT_ACCOUNT_NAME\"}"
modified_principal+="\"arn:aws:iam::$DIRECT_AWS_USER_ID:user/$DIRECT_S3_ROOT_ACCOUNT_NAME\""
else
modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/${principals[$idx]}\"}"
modified_principal+="\"arn:aws:iam::$DIRECT_AWS_USER_ID:user/${principals[$idx]}\""
fi
}
@@ -57,9 +57,13 @@ get_modified_principal() {
return 1
fi
IFS=',' read -r -a principals <<< "$1"
modified_principal=""
if [ "$DIRECT" == "true" ]; then
modified_principal="{\"AWS\": "
else
modified_principal=""
fi
if [ "${#principals[@]}" -gt 1 ]; then
modified_principal="["
modified_principal+="["
fi
for ((idx=0; idx<${#principals[@]}; idx++)); do
if [ "$DIRECT" == "true" ]; then
@@ -75,6 +79,9 @@ get_modified_principal() {
if [ "${#principals[@]}" -gt 1 ]; then
modified_principal+="]"
fi
if [ "$DIRECT" == "true" ]; then
modified_principal+="}"
fi
log 5 "modified principal: $modified_principal"
}

View File

@@ -470,7 +470,7 @@ verify_user_cant_get_object() {
return 1
fi
# shellcheck disable=SC2154
if [[ "$get_object_error" != *"Access Denied"* ]]; then
if [[ "$get_object_error" != *"AccessDenied"* ]]; then
log 2 "invalid get object error: $get_object_error"
return 1
fi
@@ -482,9 +482,9 @@ get_username_and_password() {
log 2 "'get_username_and_password' requires role, number, bucket name"
return 1
fi
if [ "$AUTOCREATE_USERS" == "true" ]; then
if [ "$AUTOGENERATE_USERS" == "true" ]; then
# NOTE: for direct users, username and password will be replaced by key ID, secret key
user_id="${USER_AUTOCREATION_PREFIX}$2"
user_id="${USER_AUTOGENERATION_PREFIX}$2"
username="$user_id"
password="abc123-${1}-${2}"
else
@@ -511,17 +511,20 @@ get_username_and_password() {
return 0
}
delete_autocreated_users() {
if [ "$USER_AUTOCREATION_PREFIX" == "" ]; then
log 5 "USER_AUTOCREATION_PREFIX must be defined to delete autocreated users"
delete_autogenerated_users() {
if [ "$USER_AUTOGENERATION_PREFIX" == "" ]; then
log 5 "USER_AUTOGENERATION_PREFIX must be defined to delete autogenerated users"
return 0
fi
list_users
if ! list_users; then
log 2 "error listing users"
return 1
fi
for user in "${parsed_users[@]}"; do
if [[ "$user" == "$USER_AUTOCREATION_PREFIX"* ]]; then
if [[ "$user" == "$USER_AUTOGENERATION_PREFIX"* ]]; then
log 5 "matched user: $user"
if ! delete_user "$user"; then
log 2 "error deleting autocreated user"
log 2 "error deleting autogenerated user"
return 1
fi
log 5 "user deletion success"
@@ -538,7 +541,7 @@ setup_user_v2() {
log 2 "error getting username and password"
return 1
fi
if [ "$AUTOCREATE_USERS" == "true" ]; then
if [ "$AUTOGENERATE_USERS" == "true" ]; then
if ! setup_user_versitygw_or_direct "$username" "$password" "$1" "$3"; then
log 2 "error setting up user"
return 1