mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-07-28 11:02:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: multiple policy-related tests, multipart upload test rewrites

Browse Source
This commit is contained in:
Luke McCrone
2024-06-27 20:19:44 -03:00
parent eafa5e12db
commit 812efe6d43
6 changed files with 315 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/commands/delete_bucket_policy.sh
+13
View File
@@ -20,4 +20,17 @@ delete_bucket_policy() {
return 1
fi
return 0
}
delete_bucket_policy_with_user() {
if [[ $# -ne 3 ]]; then
log 2 "'delete bucket policy with user' command requires bucket, username, password"
return 1
fi
if ! delete_bucket_policy_error=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api delete-bucket-policy --bucket "$1" 2>&1); then
log 2 "error deleting bucket policy: $delete_bucket_policy_error"
export delete_bucket_policy_error
return 1
fi
return 0
}
tests/commands/get_bucket_policy.sh
+20
View File
@@ -46,6 +46,26 @@ get_bucket_policy_aws() {
return 0
}
get_bucket_policy_with_user() {
if [[ $# -ne 3 ]]; then
log 2 "'get bucket policy with user' command requires bucket, username, password"
return 1
fi
if policy_json=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1); then
policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
else
if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then
bucket_policy=
else
log 2 "error getting policy for user $2: $policy_json"
return 1
fi
fi
export bucket_policy
return 0
}
get_bucket_policy_s3cmd() {
if [[ $# -ne 1 ]]; then
echo "s3cmd 'get bucket policy' command requires bucket"
tests/commands/list_multipart_uploads.sh
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
list_multipart_uploads() {
if [[ $# -ne 1 ]]; then
log 2 "'list multipart uploads' command requires bucket name"
return 1
fi
if ! uploads=$(aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
log 2 "error listing uploads: $uploads"
return 1
fi
export uploads
}
list_multipart_uploads_with_user() {
if [[ $# -ne 3 ]]; then
log 2 "'list multipart uploads' command requires bucket name, username, password"
return 1
fi
if ! uploads=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1" 2>&1); then
log 2 "error listing uploads: $uploads"
list_multipart_uploads_error=$uploads
export list_multipart_uploads_error
return 1
fi
export uploads
}
tests/commands/put_bucket_policy.sh
+16 -2
View File
@@ -2,7 +2,7 @@
put_bucket_policy() {
if [[ $# -ne 3 ]]; then
log 2 "get bucket policy command requires command type, bucket, policy file"
log 2 "'put bucket policy' command requires command type, bucket, policy file"
return 1
fi
local put_policy_result=0
@@ -23,4 +23,18 @@ put_bucket_policy() {
return 1
fi
return 0
}
}
put_bucket_policy_with_user() {
if [[ $# -ne 4 ]]; then
log 2 "'put bucket policy with user' command requires bucket, policy file, username, password"
return 1
fi
if ! policy=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" aws --no-verify-ssl s3api put-bucket-policy --bucket "$1" --policy "file://$2" 2>&1); then
log 2 "error putting bucket policy with user $3: $policy"
put_bucket_policy_error=$policy
export put_bucket_policy_error
return 1
fi
return 0
}
tests/test_aws.sh
+225 -38
View File
@@ -19,6 +19,7 @@ source ./tests/commands/get_object_legal_hold.sh
source ./tests/commands/get_object_lock_configuration.sh
source ./tests/commands/get_object_retention.sh
source ./tests/commands/get_object_tagging.sh
source ./tests/commands/list_multipart_uploads.sh
source ./tests/commands/list_object_versions.sh
source ./tests/commands/put_bucket_acl.sh
source ./tests/commands/put_bucket_policy.sh
@@ -548,29 +549,25 @@ legal_hold_retention_setup() {
@test "test-multipart-upload-list-parts" {
local bucket_file="bucket-file"
create_test_files "$bucket_file" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating test files"
create_test_files "$bucket_file" || fail "error creating test file"
dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file"
setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "failed to create bucket '$BUCKET_ONE_NAME'"
list_parts "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || list_result=$?
[[ list_result -eq 0 ]] || fail "Listing multipart upload parts failed"
list_parts "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "listing multipart upload parts failed"
declare -a parts_map
# shellcheck disable=SC2154
log 5 "parts: $parts"
for i in {0..3}; do
local part_number
local etag
part_number=$(echo "$parts" | jq ".[$i].PartNumber")
if [[ $part_number -eq "" ]]; then
echo "error: blank part number"
return 1
fi
etag=$(echo "$parts" | jq ".[$i].ETag")
if [[ $etag == "" ]]; then
echo "error: blank etag"
return 1
fi
# shellcheck disable=SC2154
part=$(echo "$parts" | grep -v "InsecureRequestWarning" | jq -r ".[$i]" 2>&1) || fail "error getting part: $part"
part_number=$(echo "$part" | jq ".PartNumber" 2>&1) || fail "error parsing part number: $part_number"
[[ $part_number != "" ]] || fail "error: blank part number"
etag=$(echo "$part" | jq ".ETag" 2>&1) || fail "error parsing etag: $etag"
[[ $etag != "" ]] || fail "error: blank etag"
# shellcheck disable=SC2004
parts_map[$part_number]=$etag
done
@@ -579,12 +576,11 @@ legal_hold_retention_setup() {
for i in {0..3}; do
local part_number
local etag
part_number=$(echo "$listed_parts" | jq ".Parts[$i].PartNumber")
etag=$(echo "$listed_parts" | jq ".Parts[$i].ETag")
if [[ ${parts_map[$part_number]} != "$etag" ]]; then
echo "error: etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)"
return 1
fi
# shellcheck disable=SC2154
listed_part=$(echo "$listed_parts" | grep -v "InsecureRequestWarning" | jq -r ".Parts[$i]" 2>&1) || fail "error parsing listed part: $listed_part"
part_number=$(echo "$listed_part" | jq ".PartNumber" 2>&1) || fail "error parsing listed part number: $part_number"
etag=$(echo "$listed_part" | jq ".ETag" 2>&1) || fail "error getting listed etag: $etag"
[[ ${parts_map[$part_number]} == "$etag" ]] || fail "error: etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)"
done
run_then_abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder/$bucket_file" 4
@@ -598,30 +594,25 @@ legal_hold_retention_setup() {
local bucket_file_two="bucket-file-two"
if [[ $RECREATE_BUCKETS == false ]]; then
abort_all_multipart_uploads "$BUCKET_ONE_NAME" || local abort_result=$?
[[ $abort_result -eq 0 ]] || fail "error aborting all uploads"
abort_all_multipart_uploads "$BUCKET_ONE_NAME" || fail "error aborting all uploads"
fi
create_test_files "$bucket_file_one" "$bucket_file_two" || local created=$?
[[ $created -eq 0 ]] || fail "Error creating test files"
setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
[[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
create_test_files "$bucket_file_one" "$bucket_file_two" || fail "error creating test files"
setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "failed to create bucket '$BUCKET_ONE_NAME'"
list_multipart_uploads "$BUCKET_ONE_NAME" "$test_file_folder"/"$bucket_file_one" "$test_file_folder"/"$bucket_file_two" || fail "failed to list multipart uploads"
create_and_list_multipart_uploads "$BUCKET_ONE_NAME" "$test_file_folder"/"$bucket_file_one" "$test_file_folder"/"$bucket_file_two" || fail "failed to list multipart uploads"
local key_one
local key_two
log 5 "$uploads"
key_one=$(echo "$uploads" | jq '.Uploads[0].Key')
key_two=$(echo "$uploads" | jq '.Uploads[1].Key')
# shellcheck disable=SC2154
log 5 "Uploads: $uploads"
raw_uploads=$(echo "$uploads" | grep -v "InsecureRequestWarning")
key_one=$(echo "$raw_uploads" | jq -r '.Uploads[0].Key' 2>&1) || fail "error getting key one: $key_one"
key_two=$(echo "$raw_uploads" | jq -r '.Uploads[1].Key' 2>&1) || fail "error getting key two: $key_two"
key_one=${key_one//\"/}
key_two=${key_two//\"/}
if [[ "$test_file_folder/$bucket_file_one" != *"$key_one" ]]; then
fail "Key mismatch ($test_file_folder/$bucket_file_one, $key_one)"
fi
if [[ "$test_file_folder/$bucket_file_two" != *"$key_two" ]]; then
fail "Key mismatch ($test_file_folder/$bucket_file_two, $key_two)"
fi
[[ "$test_file_folder/$bucket_file_one" == *"$key_one" ]] || fail "Key mismatch ($test_file_folder/$bucket_file_one, $key_one)"
[[ "$test_file_folder/$bucket_file_two" == *"$key_two" ]] || fail "Key mismatch ($test_file_folder/$bucket_file_two, $key_two)"
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files "$bucket_file_one" "$bucket_file_two"
@@ -1110,11 +1101,207 @@ EOF
fi
[[ "$put_object_error" == *"Access Denied"* ]] || fail "invalid put object error: $put_object_error"
put_object_with_user "s3api" "$test_file_folder/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$username" "$password" || fail "error putting file despite policy permissions"
if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_folder/$test_file-copy" "$username" "$password"; then
fail "able to get object without permissions"
fi
[[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
download_and_compare_file "s3api" "$test_file_folder/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_file_folder/$test_file-copy" || fail "files don't match"
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files "$test_folder/$test_file" "$test_file-copy" "$policy_file"
}
@test "test_policy_delete" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
test_file_one="test_file_one"
test_file_two="test_file_two"
username="ABCDEFG"
password="HIJKLMN"
create_test_files "$test_file_one" "$test_file_two" "$policy_file" || fail "error creating policy file, test files"
echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_file_one"
echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_file_two"
effect="Allow"
principal="$username"
action="s3:DeleteObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file_two"
if user_exists "$username"; then
delete_user "$username" || fail "failed to delete user '$username'"
fi
create_user "$username" "$password" "user" || fail "error creating user"
setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
log 5 "Policy: $(cat "$test_file_folder/$policy_file")"
put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
put_object "s3api" "$test_file_folder/$test_file_one" "$BUCKET_ONE_NAME" "$test_file_one" || fail "error copying object one"
put_object "s3api" "$test_file_folder/$test_file_two" "$BUCKET_ONE_NAME" "$test_file_two" || fail "error copying object two"
if delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_one" "$username" "$password"; then
fail "able to delete object despite lack of permissions"
fi
[[ "$delete_object_error" == *"Access Denied"* ]] || fail "invalid delete object error: $delete_object_error"
delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$username" "$password" || fail "error deleting object despite permissions"
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files "$test_file_one" "$test_file_two" "$policy_file"
}
@test "test_policy_get_bucket_policy" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
username="ABCDEFG"
password="HIJKLMN"
create_test_files "$policy_file" || fail "error creating policy file, test files"
effect="Allow"
principal="$username"
action="s3:GetBucketPolicy"
resource="arn:aws:s3:::$BUCKET_ONE_NAME"
if user_exists "$username"; then
delete_user "$username" || fail "failed to delete user '$username'"
fi
create_user "$username" "$password" "user" || fail "error creating user"
setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
if get_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then
fail "able to retrieve bucket policy despite lack of permissions"
fi
put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
get_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "error getting bucket policy despite permissions"
# shellcheck disable=SC2154
echo "$bucket_policy" > "$test_file_folder/$policy_file-copy"
log 5 "ORIG: $(cat "$test_file_folder/$policy_file")"
log 5 "COPY: $(cat "$test_file_folder/$policy_file-copy")"
compare_files "$test_file_folder/$policy_file" "$test_file_folder/$policy_file-copy" || fail "policies not equal"
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files "$policy_file" "$policy_file-copy"
}
@test "test_policy_list_multipart_uploads" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
test_file="test_file"
username="ABCDEFG"
password="HIJKLMN"
create_test_files "$policy_file" || fail "error creating policy file, test files"
create_large_file "$test_file" || error creating file "$test_file"
effect="Allow"
principal="$username"
action="s3:ListBucketMultipartUploads"
resource="arn:aws:s3:::$BUCKET_ONE_NAME"
if user_exists "$username"; then
delete_user "$username" || fail "failed to delete user '$username'"
fi
create_user "$username" "$password" "user" || fail "error creating user"
setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
create_multipart_upload "$BUCKET_ONE_NAME" "$test_file" || fail "error creating multipart upload"
if list_multipart_uploads_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then
log 2 "able to list multipart uploads despite lack of permissions"
fi
# shellcheck disable=SC2154
[[ "$list_multipart_uploads_error" == *"Access Denied"* ]] || fail "invalid list multipart uploads error: $list_multipart_uploads_error"
put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
list_multipart_uploads_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "error listing multipart uploads"
log 5 "$uploads"
upload_key=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -r ".Uploads[0].Key" 2>&1) || fail "error parsing upload key from uploads message: $upload_key"
[[ $upload_key == "$test_file" ]] || fail "upload key doesn't match file marked as being uploaded"
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files "$policy_file" "$test_file"
}
@test "test_policy_put_bucket_policy" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
policy_file_two="policy_file_two"
username="ABCDEFG"
password="HIJKLMN"
create_test_files "$policy_file" || fail "error creating policy file, test files"
effect="Allow"
principal="$username"
action="s3:PutBucketPolicy"
resource="arn:aws:s3:::$BUCKET_ONE_NAME"
if user_exists "$username"; then
delete_user "$username" || fail "failed to delete user '$username'"
fi
create_user "$username" "$password" "user" || fail "error creating user"
setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
if put_bucket_policy_with_user "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" "$username" "$password"; then
fail "able to retrieve bucket policy despite lack of permissions"
fi
put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
setup_policy_with_single_statement "$test_file_folder/$policy_file_two" "dummy" "$effect" "$principal" "s3:GetBucketPolicy" "$resource" || fail "failed to set up policy"
put_bucket_policy_with_user "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file_two" "$username" "$password" || fail "error putting bucket policy despite permissions"
get_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "error getting bucket policy despite permissions"
# shellcheck disable=SC2154
echo "$bucket_policy" > "$test_file_folder/$policy_file-copy"
log 5 "ORIG: $(cat "$test_file_folder/$policy_file_two")"
log 5 "COPY: $(cat "$test_file_folder/$policy_file-copy")"
compare_files "$test_file_folder/$policy_file_two" "$test_file_folder/$policy_file-copy" || fail "policies not equal"
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files "$policy_file" "$policy_file_two" "$policy_file-copy"
}
@test "test_policy_delete_bucket_policy" {
# TODO (https://github.com/versity/versitygw/issues/637)
if [[ $RECREATE_BUCKETS == "false" ]]; then
return 0
fi
policy_file="policy_file"
username="ABCDEFG"
password="HIJKLMN"
create_test_files "$policy_file" || fail "error creating policy file, test files"
effect="Allow"
principal="$username"
action="s3:DeleteBucketPolicy"
resource="arn:aws:s3:::$BUCKET_ONE_NAME"
if user_exists "$username"; then
delete_user "$username" || fail "failed to delete user '$username'"
fi
create_user "$username" "$password" "user" || fail "error creating user"
setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
if delete_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password"; then
fail "able to delete bucket policy with user $username without right permissions"
fi
setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
delete_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "unable to delete bucket policy"
delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
delete_test_files "$policy_file"
}
# ensure that lists of files greater than a size of 1000 (pagination) are returned properly
#@test "test_list_objects_file_count" {
# test_common_list_objects_file_count "aws"
tests/util.sh
+14 -19
View File
@@ -867,19 +867,17 @@ copy_file() {
# export parts on success, return 1 for error
list_parts() {
if [ $# -ne 4 ]; then
echo "list multipart upload parts command missing bucket, key, file, and/or part count"
log 2 "list multipart upload parts command requires bucket, key, file, and part count"
return 1
fi
multipart_upload_before_completion "$1" "$2" "$3" "$4" || result=$?
if [[ $result -ne 0 ]]; then
echo "error performing pre-completion multipart upload"
if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then
log 2 "error performing pre-completion multipart upload"
return 1
fi
listed_parts=$(aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$upload_id") || local listed=$?
if [[ $listed -ne 0 ]]; then
echo "Error aborting upload: $parts"
if ! listed_parts=$(aws --no-verify-ssl s3api list-parts --bucket "$1" --key "$2" --upload-id "$upload_id" 2>&1); then
log 2 "Error listing multipart upload parts: $listed_parts"
return 1
fi
export listed_parts
@@ -888,30 +886,27 @@ list_parts() {
# list unfinished multipart uploads
# params: bucket, key one, key two
# export current two uploads on success, return 1 for error
list_multipart_uploads() {
create_and_list_multipart_uploads() {
if [ $# -ne 3 ]; then
echo "list multipart uploads command requires bucket and two keys"
log 2 "list multipart uploads command requires bucket and two keys"
return 1
fi
create_multipart_upload "$1" "$2" || local create_result=$?
if [[ $create_result -ne 0 ]]; then
echo "error creating multpart upload"
if ! create_multipart_upload "$1" "$2"; then
log 2 "error creating multpart upload"
return 1
fi
create_multipart_upload "$1" "$3" || local create_result_two=$?
if [[ $create_result_two -ne 0 ]]; then
echo "error creating multpart upload two"
if ! create_multipart_upload "$1" "$3"; then
log 2 "error creating multpart upload two"
return 1
fi
uploads=$(aws --no-verify-ssl s3api list-multipart-uploads --bucket "$1") || local list_result=$?
if [[ $list_result -ne 0 ]]; then
echo "error listing uploads: $uploads"
if ! list_multipart_uploads "$1"; then
echo "error listing uploads"
return 1
fi
export uploads
return 0
}
multipart_upload_from_bucket() {