diff --git a/tests/commands/copy_object.sh b/tests/commands/copy_object.sh new file mode 100644 index 00000000..e717705c --- /dev/null +++ b/tests/commands/copy_object.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +copy_object() { + if [ $# -ne 3 ]; then + echo "copy object command requires command type, source, destination" + return 1 + fi + local exit_code=0 + local error + if [[ $1 == 'aws' ]]; then + error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3" 2>&1) || exit_code=$? + elif [[ $1 == 's3cmd' ]]; then + error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$(dirname "$3")" 2>&1) || exit_code=$? + elif [[ $1 == 'mc' ]]; then + error=$(mc --insecure cp "$2" "$MC_ALIAS"/"$(dirname "$3")" 2>&1) || exit_code=$? + else + echo "invalid command type $1" + return 1 + fi + log 5 "copy object exit code: $exit_code" + if [ $exit_code -ne 0 ]; then + echo "error copying object to bucket: $error" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/commands/delete_bucket_policy.sh b/tests/commands/delete_bucket_policy.sh new file mode 100644 index 00000000..ed98e299 --- /dev/null +++ b/tests/commands/delete_bucket_policy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +delete_bucket_policy() { + if [[ $# -ne 2 ]]; then + echo "delete bucket policy command requires command type, bucket" + return 1 + fi + if [[ $1 == 'aws' ]]; then + error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2") || delete_result=$? + elif [[ $1 == 's3cmd' ]]; then + error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2") || delete_result=$? + elif [[ $1 == 'mc' ]]; then + error=$(mc --insecure anonymous set none "$MC_ALIAS/$2") || delete_result=$? + else + echo "command 'get bucket policy' not implemented for '$1'" + return 1 + fi + if [[ $delete_result -ne 0 ]]; then + echo "error deleting bucket policy: $error" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/commands/get_bucket_policy.sh b/tests/commands/get_bucket_policy.sh new file mode 100644 index 00000000..099e8c3e --- /dev/null +++ b/tests/commands/get_bucket_policy.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +get_bucket_policy() { + if [[ $# -ne 2 ]]; then + echo "get bucket policy command requires command type, bucket" + return 1 + fi + if [[ $1 == 'aws' ]]; then + get_bucket_policy_aws "$2" || get_result=$? + elif [[ $1 == 's3cmd' ]]; then + get_bucket_policy_s3cmd "$2" || get_result=$? + elif [[ $1 == 'mc' ]]; then + get_bucket_policy_mc "$2" || get_result=$? + else + echo "command 'get bucket policy' not implemented for '$1'" + return 1 + fi + if [[ $get_result -ne 0 ]]; then + echo "error getting policy: $bucket_policy" + return 1 + fi + export bucket_policy + return 0 +} + +get_bucket_policy_aws() { + if [[ $# -ne 1 ]]; then + echo "aws 'get bucket policy' command requires bucket" + return 1 + fi + policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1") || get_result=$? + if [[ $get_result -ne 0 ]]; then + echo "error getting policy: $policy_json" + return 1 + fi + bucket_policy=$(echo "$policy_json" | jq -r '.Policy') + export bucket_policy + return 0 +} + +get_bucket_policy_s3cmd() { + if [[ $# -ne 1 ]]; then + echo "s3cmd 'get bucket policy' command requires bucket" + return 1 + fi + + info=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate info "s3://$1") || get_result=$? + if [[ $get_result -ne 0 ]]; then + echo "error getting bucket policy: $info" + return 1 + fi + + bucket_policy="" + policy_brackets=false + while IFS= read -r line; do + if [[ $policy_brackets == false ]]; then + policy_line=$(echo "$line" | grep 'Policy: ') + if [[ $policy_line != "" ]]; then + if [[ $policy_line != *'{' ]]; then + break + fi + policy_brackets=true + bucket_policy+="{" + fi + else + bucket_policy+=$line + if [[ $line == "" ]]; then + break + fi + fi + done <<< "$info" + export bucket_policy + return 0 +} + +get_bucket_policy_mc() { + if [[ $# -ne 1 ]]; then + echo "aws 'get bucket policy' command requires bucket" + return 1 + fi + bucket_policy=$(mc --insecure anonymous get-json "$MC_ALIAS/$1") || get_result=$? + if [[ $get_result -ne 0 ]]; then + echo "error getting policy: $bucket_policy" + return 1 + fi + export bucket_policy + return 0 +} \ No newline at end of file diff --git a/tests/commands/put_bucket_policy.sh b/tests/commands/put_bucket_policy.sh new file mode 100644 index 00000000..c57efe23 --- /dev/null +++ b/tests/commands/put_bucket_policy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +put_bucket_policy() { + if [[ $# -ne 3 ]]; then + echo "get bucket policy command requires command type, bucket, policy file" + return 1 + fi + if [[ $1 == 'aws' ]]; then + policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3") || get_result=$? + elif [[ $1 == 's3cmd' ]]; then + policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2") || get_result=$? + elif [[ $1 == 'mc' ]]; then + policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2") + else + echo "command 'put bucket policy' not implemented for '$1'" + return 1 + fi + if [[ $get_result -ne 0 ]]; then + echo "error putting policy: $policy" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/test_aws.sh b/tests/test_aws.sh index 502c60c3..a54a7d41 100755 --- a/tests/test_aws.sh +++ b/tests/test_aws.sh @@ -6,7 +6,75 @@ source ./tests/util_aws.sh source ./tests/util_bucket_create.sh source ./tests/util_file.sh source ./tests/test_common.sh +source ./tests/commands/copy_object.sh +source ./tests/commands/delete_bucket_policy.sh source ./tests/commands/delete_object_tagging.sh +source ./tests/commands/get_bucket_policy.sh +source ./tests/commands/put_bucket_policy.sh + +@test "test_abort_multipart_upload" { + local bucket_file="bucket-file" + bucket_file_data="test file\n" + + create_test_files "$bucket_file" || local created=$? + printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file + [[ $created -eq 0 ]] || fail "Error creating test files" + setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? + [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" + + abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$? + [[ $abort_result -eq 0 ]] || fail "Abort failed" + + object_exists "aws" "$BUCKET_ONE_NAME/$bucket_file" || exists=$? + [[ $exists -eq 1 ]] || fail "Upload file exists after abort" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files $bucket_file +} + +@test "test_complete_multipart_upload" { + local bucket_file="bucket-file" + bucket_file_data="test file\n" + + create_test_files "$bucket_file" || local created=$? + printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file + [[ $created -eq 0 ]] || fail "Error creating test files" + setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? + [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" + + multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$? + [[ $upload_result -eq 0 ]] || fail "Error performing multipart upload" + + copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy" + compare_files "$test_file_folder/$bucket_file-copy" "$test_file_folder"/$bucket_file || compare_result=$? + [[ $compare_result -eq 0 ]] || fail "Files do not match" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_test_files $bucket_file +} + +@test "test_copy_object" { + bucket_file="bucket_file" + + create_test_files "$bucket_file" || local created=$? + [[ $created -eq 0 ]] || fail "Error creating test files" + setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? + [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + setup_bucket "aws" "$BUCKET_TWO_NAME" || local setup_result_two=$? + [[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error" + copy_object "aws" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || local copy_result=$? + [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" + error=$(aws --no-verify-ssl s3api copy-object --copy-source "$BUCKET_ONE_NAME"/"$bucket_file" --key "$bucket_file" --bucket "$BUCKET_TWO_NAME" 2>&1) || local copy_result=$? + [[ $copy_result -eq 0 ]] || fail "Error copying file: $error" + copy_file "s3://$BUCKET_TWO_NAME"/"$bucket_file" "$test_file_folder/${bucket_file}_copy" || local copy_result=$? + [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" + compare_files "$test_file_folder/$bucket_file" "$test_file_folder/${bucket_file}_copy" || local compare_result=$? + [[ $compare_result -eq 0 ]] || file "files don't match" + + delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + delete_bucket_or_contents "aws" "$BUCKET_TWO_NAME" + delete_test_files "$bucket_file" +} # test creation and deletion of bucket on versitygw @test "test_create_delete_bucket_aws" { @@ -27,12 +95,12 @@ source ./tests/commands/delete_object_tagging.sh } # test adding and removing an object on versitygw -@test "test_put_object-with-data" { - test_common_put_object_with_data "aws" +@test "test_copy_object_with_data" { + test_common_copy_object_with_data "aws" } -@test "test_put_object-no-data" { - test_common_put_object_no_data "aws" +@test "test_copy_object_no_data" { + test_common_copy_object_no_data "aws" } # test listing buckets on versitygw @@ -81,7 +149,6 @@ source ./tests/commands/delete_object_tagging.sh # delete_bucket_or_contents "$BUCKET_ONE_NAME" #} - # test ability to delete multiple objects from bucket @test "test_delete_objects" { local object_one="test-file-one" @@ -92,9 +159,9 @@ source ./tests/commands/delete_object_tagging.sh setup_bucket "aws" "$BUCKET_ONE_NAME" || local result_one=$? [[ $result_one -eq 0 ]] || fail "Error creating bucket" - put_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$? + copy_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$? [[ $result_two -eq 0 ]] || fail "Error adding object one" - put_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$? + copy_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$? [[ $result_three -eq 0 ]] || fail "Error adding object two" error=$(aws --no-verify-ssl s3api delete-objects --bucket "$BUCKET_ONE_NAME" --delete '{ @@ -130,10 +197,10 @@ source ./tests/commands/delete_object_tagging.sh printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two" setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - put_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$? - [[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one" - put_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$? - [[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two" + copy_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local copy_result_one=$? + [[ $copy_result_one -eq 0 ]] || fail "Failed to add object $object_one" + copy_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local copy_result_two=$? + [[ $copy_result_two -eq 0 ]] || fail "Failed to add object $object_two" list_objects_s3api_v1 "$BUCKET_ONE_NAME" key_one=$(echo "$objects" | jq '.Contents[0].Key') @@ -160,10 +227,10 @@ source ./tests/commands/delete_object_tagging.sh printf "%s" "$object_two_data" > "$test_file_folder"/"$object_two" setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - put_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local put_object_one=$? - [[ $put_object_one -eq 0 ]] || fail "Failed to add object $object_one" - put_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local put_object_two=$? - [[ $put_object_two -eq 0 ]] || fail "Failed to add object $object_two" + copy_object "aws" "$test_file_folder"/"$object_one" "$BUCKET_ONE_NAME"/"$object_one" || local copy_object_one=$? + [[ $copy_object_one -eq 0 ]] || fail "Failed to add object $object_one" + copy_object "aws" "$test_file_folder"/"$object_two" "$BUCKET_ONE_NAME"/"$object_two" || local copy_object_two=$? + [[ $copy_object_two -eq 0 ]] || fail "Failed to add object $object_two" list_objects_s3api_v2 "$BUCKET_ONE_NAME" key_one=$(echo "$objects" | jq '.Contents[0].Key') @@ -184,50 +251,6 @@ source ./tests/commands/delete_object_tagging.sh test_common_set_get_object_tags "aws" } -# test multi-part upload -@test "test-multi-part-upload" { - local bucket_file="bucket-file" - bucket_file_data="test file\n" - - create_test_files "$bucket_file" || local created=$? - printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file - [[ $created -eq 0 ]] || fail "Error creating test files" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? - [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - - multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$? - [[ $upload_result -eq 0 ]] || fail "Error performing multipart upload" - - copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy" - compare_files "$test_file_folder/$bucket_file-copy" "$test_file_folder"/$bucket_file || compare_result=$? - [[ $compare_result -eq 0 ]] || fail "Files do not match" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files $bucket_file -} - -# ADDED -# test multi-part upload abort -@test "test-multi-part-upload-abort" { - local bucket_file="bucket-file" - bucket_file_data="test file\n" - - create_test_files "$bucket_file" || local created=$? - printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file - [[ $created -eq 0 ]] || fail "Error creating test files" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? - [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - - abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$? - [[ $abort_result -eq 0 ]] || fail "Abort failed" - - object_exists "aws" "$BUCKET_ONE_NAME/$bucket_file" || exists=$? - [[ $exists -eq 1 ]] || fail "Upload file exists after abort" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files $bucket_file -} - # test multi-part upload list parts command @test "test-multipart-upload-list-parts" { local bucket_file="bucket-file" @@ -347,8 +370,8 @@ source ./tests/commands/delete_object_tagging.sh setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - put_object "aws" "$test_file_folder"/"$folder_name"/"$object_name" "$BUCKET_ONE_NAME"/"$folder_name"/"$object_name" || local put_object=$? - [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" + copy_object "aws" "$test_file_folder"/"$folder_name"/"$object_name" "$BUCKET_ONE_NAME"/"$folder_name"/"$object_name" || local copy_result=$? + [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/" prefix=$(echo "${objects[@]}" | jq ".CommonPrefixes[0].Prefix") @@ -397,30 +420,6 @@ source ./tests/commands/delete_object_tagging.sh delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" } -@test "test_copy_object_aws" { - - bucket_file="bucket_file" - - create_test_files "$bucket_file" || local created=$? - [[ $created -eq 0 ]] || fail "Error creating test files" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local setup_result=$? - [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - setup_bucket "aws" "$BUCKET_TWO_NAME" || local setup_result_two=$? - [[ $setup_result_two -eq 0 ]] || fail "Bucket two setup error" - put_object "aws" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || local put_object=$? - [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" - error=$(aws --no-verify-ssl s3api copy-object --copy-source "$BUCKET_ONE_NAME"/"$bucket_file" --key "$bucket_file" --bucket "$BUCKET_TWO_NAME" 2>&1) || local copy_result=$? - [[ $copy_result -eq 0 ]] || fail "Error copying file: $error" - copy_file "s3://$BUCKET_TWO_NAME"/"$bucket_file" "$test_file_folder/${bucket_file}_copy" || local put_object=$? - [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" - compare_files "$test_file_folder/$bucket_file" "$test_file_folder/${bucket_file}_copy" || local compare_result=$? - [[ $compare_result -eq 0 ]] || file "files don't match" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_bucket_or_contents "aws" "$BUCKET_TWO_NAME" - delete_test_files "$bucket_file" -} - @test "test_add_object_metadata" { object_one="object-one" @@ -434,8 +433,8 @@ source ./tests/commands/delete_object_tagging.sh [[ $setup_result -eq 0 ]] || fail "error setting up bucket" object="$test_file_folder"/"$object_one" - put_object_with_metadata "aws" "$object" "$BUCKET_ONE_NAME" "$test_key" "$test_value" || put_object=$? - [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" + put_object_with_metadata "aws" "$object" "$BUCKET_ONE_NAME" "$test_key" "$test_value" || copy_result=$? + [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" object_exists "aws" "$object" || local exists_result_one=$? [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket" @@ -453,4 +452,8 @@ source ./tests/commands/delete_object_tagging.sh @test "test_get_bucket_location" { test_common_get_bucket_location "aws" +} + +@test "test_get_put_delete_bucket_policy" { + test_common_get_put_delete_bucket_policy "aws" } \ No newline at end of file diff --git a/tests/test_aws_posix.sh b/tests/test_aws_posix.sh index c787734c..0f330cad 100755 --- a/tests/test_aws_posix.sh +++ b/tests/test_aws_posix.sh @@ -5,6 +5,7 @@ source ./tests/util.sh source ./tests/util_bucket_create.sh source ./tests/util_file.sh source ./tests/util_posix.sh +source ./tests/commands/copy_object.sh # test that changes to local folders and files are reflected on S3 @test "test_local_creation_deletion" { @@ -51,7 +52,7 @@ source ./tests/util_posix.sh fi setup_bucket "aws" "$bucket_name" || local created=$? [[ $created -eq 0 ]] || fail "Error creating bucket" - put_object "aws" "$test_file_folder"/"$object_name" "$bucket_name"/"$object_name" || local result="$?" + copy_object "aws" "$test_file_folder"/"$object_name" "$bucket_name"/"$object_name" || local result="$?" [[ result -eq 0 ]] || fail "Error adding object one" chmod 000 "$LOCAL_FOLDER"/"$bucket_name"/$object_name diff --git a/tests/test_common.sh b/tests/test_common.sh index 651a3d91..701f563e 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -2,10 +2,32 @@ source ./tests/util.sh source ./tests/util_file.sh +source ./tests/util_policy.sh +source ./tests/commands/copy_object.sh source ./tests/commands/delete_object_tagging.sh source ./tests/commands/get_bucket_location.sh source ./tests/commands/get_bucket_tagging.sh +test_common_multipart_upload() { + if [[ $# -ne 1 ]]; then + echo "multipart upload command missing command type" + return 1 + fi + bucket_file="largefile" + + create_large_file "$bucket_file" || local created=$? + [[ $created -eq 0 ]] || fail "Error creating test file for multipart upload" + + setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? + [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" + + copy_object "$1" "$test_file_folder"/$bucket_file "$BUCKET_ONE_NAME/$bucket_file" || local put_result=$? + [[ $put_result -eq 0 ]] || fail "failed to copy file" + + delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" + delete_test_files $bucket_file +} + # common test for creating, deleting buckets # param: "aws" or "s3cmd" # pass if buckets are properly listed, fail if not @@ -28,7 +50,7 @@ test_common_create_delete_bucket() { [[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket" } -test_common_put_object_with_data() { +test_common_copy_object_with_data() { if [[ $# -ne 1 ]]; then fail "put object test requires command type" fi @@ -37,10 +59,10 @@ test_common_put_object_with_data() { create_test_files "$object_name" || local create_result=$? [[ $create_result -eq 0 ]] || fail "Error creating test file" echo "test data" > "$test_file_folder"/"$object_name" - test_common_put_object "$1" "$object_name" + test_common_copy_object "$1" "$object_name" } -test_common_put_object_no_data() { +test_common_copy_object_no_data() { if [[ $# -ne 1 ]]; then fail "put object test requires command type" fi @@ -48,10 +70,10 @@ test_common_put_object_no_data() { local object_name="test-object" create_test_files "$object_name" || local create_result=$? [[ $create_result -eq 0 ]] || fail "Error creating test file" - test_common_put_object "$1" "$object_name" + test_common_copy_object "$1" "$object_name" } -test_common_put_object() { +test_common_copy_object() { if [[ $# -ne 2 ]]; then fail "put object test requires command type, file" fi @@ -60,8 +82,8 @@ test_common_put_object() { [[ $setup_result -eq 0 ]] || fail "error setting up bucket" object="$BUCKET_ONE_NAME"/"$2" - put_object "$1" "$test_file_folder"/"$2" "$object" || local put_object=$? - [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" + copy_object "$1" "$test_file_folder"/"$2" "$object" || local copy_result=$? + [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" object_exists "$1" "$object" || local exists_result_one=$? [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket" @@ -126,9 +148,9 @@ test_common_list_objects() { echo "test data 2" > "$test_file_folder"/"$object_two" setup_bucket "$1" "$BUCKET_ONE_NAME" || local result_one=$? [[ result_one -eq 0 ]] || fail "Error creating bucket" - put_object "$1" "$test_file_folder"/$object_one "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$? + copy_object "$1" "$test_file_folder"/$object_one "$BUCKET_ONE_NAME"/"$object_one" || local result_two=$? [[ result_two -eq 0 ]] || fail "Error adding object one" - put_object "$1" "$test_file_folder"/$object_two "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$? + copy_object "$1" "$test_file_folder"/$object_two "$BUCKET_ONE_NAME"/"$object_two" || local result_three=$? [[ result_three -eq 0 ]] || fail "Error adding object two" list_objects "$1" "$BUCKET_ONE_NAME" @@ -210,8 +232,8 @@ test_common_set_get_object_tags() { setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" local object_path="$BUCKET_ONE_NAME"/"$bucket_file" - put_object "$1" "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$? - [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'" + copy_object "$1" "$test_file_folder"/"$bucket_file" "$object_path" || local copy_result=$? + [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'" get_object_tags "$1" "$BUCKET_ONE_NAME" $bucket_file || local get_result=$? [[ $get_result -eq 0 ]] || fail "Error getting object tags" @@ -240,26 +262,6 @@ test_common_set_get_object_tags() { delete_test_files $bucket_file } -test_common_multipart_upload() { - if [[ $# -ne 1 ]]; then - echo "multipart upload command missing command type" - return 1 - fi - bucket_file="largefile" - - create_large_file "$bucket_file" || local created=$? - [[ $created -eq 0 ]] || fail "Error creating test file for multipart upload" - - setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? - [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - - put_object "$1" "$test_file_folder"/$bucket_file "$BUCKET_ONE_NAME/$bucket_file" || local put_result=$? - [[ $put_result -eq 0 ]] || fail "failed to copy file" - - delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" - delete_test_files $bucket_file -} - test_common_presigned_url_utf8_chars() { if [[ $# -ne 1 ]]; then @@ -276,7 +278,7 @@ test_common_presigned_url_utf8_chars() { setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - put_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || put_result=$? + copy_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || put_result=$? [[ $put_result -eq 0 ]] || fail "Failed to add object $bucket_file" create_presigned_url "$1" "$BUCKET_ONE_NAME" "$bucket_file" || presigned_result=$? @@ -331,8 +333,8 @@ test_common_delete_object_tagging() { setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$? [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - put_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || local put_object=$? - [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" + copy_object "$1" "$test_file_folder"/"$bucket_file" "$BUCKET_ONE_NAME"/"$bucket_file" || local copy_result=$? + [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket" put_object_tag "$1" "$BUCKET_ONE_NAME" "$bucket_file" "$tag_key" "$tag_value" || put_result=$? [[ $put_result -eq 0 ]] || fail "failed to add tags to object" @@ -358,3 +360,60 @@ test_common_get_bucket_location() { # shellcheck disable=SC2154 [[ $bucket_location == "null" ]] || [[ $bucket_location == "us-east-1" ]] || fail "wrong location: '$bucket_location'" } + +test_common_get_put_delete_bucket_policy() { + [[ $# -eq 1 ]] || fail "get/put/delete policy test requires command type" + + policy_file="policy_file" + + create_test_files "$policy_file" || local created=$? + [[ $created -eq 0 ]] || fail "Error creating policy file" + + effect="Allow" + principal="*" + action="s3:GetObject" + resource="arn:aws:s3:::$BUCKET_ONE_NAME/*" + + cat < "$test_file_folder"/$policy_file + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "$effect", + "Principal": "$principal", + "Action": "$action", + "Resource": "$resource" + } + ] + } +EOF + + setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$? + [[ $setup_result -eq 0 ]] || fail "error setting up bucket" + + check_for_empty_policy "$1" "$BUCKET_ONE_NAME" || check_result=$? + [[ $get_result -eq 0 ]] || fail "policy not empty" + + put_bucket_policy "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$policy_file" || put_result=$? + [[ $put_result -eq 0 ]] || fail "error putting bucket" + + get_bucket_policy "$1" "$BUCKET_ONE_NAME" || get_result=$? + [[ $get_result -eq 0 ]] || fail "error getting bucket policy after setting" + + returned_effect=$(echo "$bucket_policy" | jq -r '.Statement[0].Effect') + [[ $effect == "$returned_effect" ]] || fail "effect mismatch ($effect, $returned_effect)" + returned_principal=$(echo "$bucket_policy" | jq -r '.Statement[0].Principal') + [[ $principal == "$returned_principal" ]] || fail "principal mismatch ($principal, $returned_principal)" + returned_action=$(echo "$bucket_policy" | jq -r '.Statement[0].Action') + [[ $action == "$returned_action" ]] || fail "action mismatch ($action, $returned_action)" + returned_resource=$(echo "$bucket_policy" | jq -r '.Statement[0].Resource') + [[ $resource == "$returned_resource" ]] || fail "resource mismatch ($resource, $returned_resource)" + + delete_bucket_policy "$1" "$BUCKET_ONE_NAME" || delete_result=$? + [[ $delete_result -eq 0 ]] || fail "error deleting policy" + + check_for_empty_policy "$1" "$BUCKET_ONE_NAME" || check_result=$? + [[ $get_result -eq 0 ]] || fail "policy not empty after deletion" + + delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" +} diff --git a/tests/test_mc.sh b/tests/test_mc.sh index d47fa646..3ad4e2ba 100755 --- a/tests/test_mc.sh +++ b/tests/test_mc.sh @@ -3,20 +3,27 @@ source ./tests/test_common.sh source ./tests/setup.sh source ./tests/util_bucket_create.sh +source ./tests/commands/delete_bucket_policy.sh +source ./tests/commands/get_bucket_policy.sh +source ./tests/commands/put_bucket_policy.sh export RUN_MC=true +@test "test_multipart_upload_mc" { + test_common_multipart_upload "mc" +} + # test mc bucket creation/deletion @test "test_create_delete_bucket_mc" { test_common_create_delete_bucket "mc" } @test "test_put_object-with-data-mc" { - test_common_put_object_with_data "mc" + test_common_copy_object_with_data "mc" } @test "test_put_object-no-data-mc" { - test_common_put_object_no_data "mc" + test_common_copy_object_no_data "mc" } @test "test_list_buckets_mc" { @@ -35,10 +42,6 @@ export RUN_MC=true test_common_set_get_object_tags "mc" } -@test "test_multipart_upload_mc" { - test_common_multipart_upload "mc" -} - @test "test_presigned_url_utf8_chars_mc" { test_common_presigned_url_utf8_chars "mc" } @@ -84,3 +87,7 @@ export RUN_MC=true @test "test_get_bucket_location" { test_common_get_bucket_location "mc" } + +@test "test_get_put_delete_bucket_policy" { + test_common_get_put_delete_bucket_policy "mc" +} diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 679cd8de..4eb77451 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -4,21 +4,28 @@ source ./tests/setup.sh source ./tests/test_common.sh source ./tests/util.sh source ./tests/util_bucket_create.sh +source ./tests/commands/delete_bucket_policy.sh +source ./tests/commands/get_bucket_policy.sh +source ./tests/commands/put_bucket_policy.sh export RUN_S3CMD=true +@test "test_multipart_upload_s3cmd" { + test_common_multipart_upload "s3cmd" +} + # test s3cmd bucket creation/deletion @test "test_create_delete_bucket_s3cmd" { test_common_create_delete_bucket "s3cmd" } # test s3cmd put object -@test "test_put_object_with_data_s3cmd" { - test_common_put_object_with_data "s3cmd" +@test "test_copy_object_with_data" { + test_common_copy_object_with_data "s3cmd" } -@test "test_put_object_no_data_s3cmd" { - test_common_put_object_no_data "s3cmd" +@test "test_copy_object_no_data" { + test_common_copy_object_no_data "s3cmd" } # test listing buckets on versitygw @@ -30,10 +37,6 @@ export RUN_S3CMD=true test_common_list_objects "s3cmd" } -@test "test_multipart_upload_s3cmd" { - test_common_multipart_upload "s3cmd" -} - #@test "test_presigned_url_utf8_chars_s3cmd" { # test_common_presigned_url_utf8_chars "s3cmd" #} @@ -75,3 +78,7 @@ export RUN_S3CMD=true @test "test_get_bucket_location" { test_common_get_bucket_location "s3cmd" } + +@test "test_get_put_delete_bucket_policy" { + test_common_get_put_delete_bucket_policy "s3cmd" +} diff --git a/tests/util.sh b/tests/util.sh index eb89019e..40393680 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -228,34 +228,6 @@ object_exists() { return 0 } -# add object to versitygw -# params: source file, destination copy location -# return 0 for success, 1 for failure -put_object() { - if [ $# -ne 3 ]; then - echo "put object command requires command type, source, destination" - return 1 - fi - local exit_code=0 - local error - if [[ $1 == 'aws' ]]; then - error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3" 2>&1) || exit_code=$? - elif [[ $1 == 's3cmd' ]]; then - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$(dirname "$3")" 2>&1) || exit_code=$? - elif [[ $1 == 'mc' ]]; then - error=$(mc --insecure cp "$2" "$MC_ALIAS"/"$(dirname "$3")" 2>&1) || exit_code=$? - else - echo "invalid command type $1" - return 1 - fi - log 5 "put object exit code: $exit_code" - if [ $exit_code -ne 0 ]; then - echo "error copying object to bucket: $error" - return 1 - fi - return 0 -} - put_object_with_metadata() { if [ $# -ne 5 ]; then echo "put object command requires command type, source, destination, key, value" @@ -345,8 +317,8 @@ check_and_put_object() { return 1 fi if [ "$exists_result" -eq 1 ]; then - put_object "$1" "$2" || local put_result=$? - if [ "$put_result" -ne 0 ]; then + copy_object "$1" "$2" || local copy_result=$? + if [ "$copy_result" -ne 0 ]; then echo "error adding object" return 1 fi @@ -1004,9 +976,9 @@ multipart_upload_from_bucket() { fi for ((i=0;i<$4;i++)) { - put_object "aws" "$3"-"$i" "$1" || put_result=$? - if [[ $put_result -ne 0 ]]; then - echo "error putting object" + copy_object "aws" "$3"-"$i" "$1" || copy_result=$? + if [[ $copy_result -ne 0 ]]; then + echo "error copying object" return 1 fi } diff --git a/tests/util_policy.sh b/tests/util_policy.sh new file mode 100644 index 00000000..d911bad7 --- /dev/null +++ b/tests/util_policy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +check_for_empty_policy() { + if [[ $# -ne 2 ]]; then + echo "check for empty policy command requires command type, bucket name" + return 1 + fi + + get_bucket_policy "$1" "$2" || get_result=$? + if [[ $get_result -ne 0 ]]; then + echo "error getting bucket policy" + return 1 + fi + + # shellcheck disable=SC2154 + policy=$(echo "$bucket_policy" | jq -r '.Policy') + statement=$(echo "$policy" | jq -r '.Statement[0]') + if [[ "" != "$statement" ]] && [[ "null" != "$statement" ]]; then + echo "policy should be empty (actual value: '$statement')" + return 1 + fi + return 0 +} \ No newline at end of file