From 89b4b615ab5b2a308de57b833a47c9b5a362d5b9 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Fri, 2 Feb 2024 16:26:37 -0300 Subject: [PATCH] test: cmdline tests (acls, get bucket/object info) --- tests/posix_tests.sh | 151 +++++++------------ tests/s3_bucket_tests.sh | 265 ++++++---------------------------- tests/{tests.sh => setup.sh} | 0 tests/util.sh | 273 +++++++++++++++++++++++++++++++++++ tests/util_posix.sh | 97 +++++++++++++ 5 files changed, 471 insertions(+), 315 deletions(-) mode change 100644 => 100755 tests/posix_tests.sh rename tests/{tests.sh => setup.sh} (100%) create mode 100644 tests/util.sh create mode 100644 tests/util_posix.sh diff --git a/tests/posix_tests.sh b/tests/posix_tests.sh old mode 100644 new mode 100755 index 56b5c87..2dbbdd4 --- a/tests/posix_tests.sh +++ b/tests/posix_tests.sh @@ -1,105 +1,11 @@ #!/usr/bin/env bats -source ./tests/tests.sh - -# check if object exists both on S3 and locally -# param: object path -# 0 for yes, 1 for no, 2 for error -object_exists_remote_and_local() { - if [ $# -ne 1 ]; then - echo "object existence check requires single name parameter" - return 2 - fi - object_exists "$1" || local exist_result=$? - if [[ $exist_result -eq 2 ]]; then - echo "Error checking if object exists" - return 2 - fi - if [[ $exist_result -eq 1 ]]; then - echo "Error: object doesn't exist remotely" - return 1 - fi - if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then - echo "Error: object doesn't exist locally" - return 1 - fi - return 0 -} - -# check if object doesn't exist both on S3 and locally -# param: object path -# return 0 for doesn't exist, 1 for still exists, 2 for error -object_not_exists_remote_and_local() { - if [ $# -ne 1 ]; then - echo "object non-existence check requires single name parameter" - return 2 - fi - object_exists "$1" || local exist_result=$? - if [[ $exist_result -eq 2 ]]; then - echo "Error checking if object doesn't exist" - return 2 - fi - if [[ $exist_result -eq 0 ]]; then - echo "Error: object exists remotely" - return 1 - fi - if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then - echo "Error: object exists locally" - return 1 - fi - return 0 -} - -# check if a bucket doesn't exist both on S3 and on gateway -# param: bucket name -# return: 0 for doesn't exist, 1 for does, 2 for error -bucket_not_exists_remote_and_local() { - if [ $# -ne 1 ]; then - echo "bucket existence check requires single name parameter" - return 2 - fi - bucket_exists "$1" || local exist_result=$? - if [[ $exist_result -eq 2 ]]; then - echo "Error checking if bucket exists" - return 2 - fi - if [[ $exist_result -eq 0 ]]; then - echo "Error: bucket exists remotely" - return 1 - fi - if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then - echo "Error: bucket exists locally" - return 1 - fi - return 0 -} - -# check if a bucket exists both on S3 and on gateway -# param: bucket name -# return: 0 for yes, 1 for no, 2 for error -bucket_exists_remote_and_local() { - if [ $# -ne 1 ]; then - echo "bucket existence check requires single name parameter" - return 2 - fi - bucket_exists "$1" || local exist_result=$? - if [[ $exist_result -eq 2 ]]; then - echo "Error checking if bucket exists" - return 2 - fi - if [[ $exist_result -eq 1 ]]; then - echo "Error: bucket doesn't exist remotely" - return 1 - fi - if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then - echo "Error: bucket doesn't exist locally" - return 1 - fi - return 0 -} +source ./tests/setup.sh +source ./tests/util.sh +source ./tests/util_posix.sh # test that changes to local folders and files are reflected on S3 -@test test_local_creation_deletion { +@test "test_local_creation_deletion" { local bucket_name="versity-gwtest-put-object-test" local object_name="test-object" @@ -137,3 +43,52 @@ bucket_exists_remote_and_local() { [[ $bucket_deleted -eq 0 ]] || fail "Failed bucket deletion check" } +# test head-object command +@test "test_head_object" { + + local bucket_name="versity-gwtest-head-object" + local object_name="object-one" + + touch "$object_name" + + if [ -e "$LOCAL_FOLDER"/$bucket_name/$object_name ]; then + chmod 755 "$LOCAL_FOLDER"/$bucket_name/$object_name + fi + check_and_create_bucket $bucket_name || local created=$? + [[ $created -eq 0 ]] || fail "Error creating bucket" + put_object "$object_name" "$bucket_name"/"$object_name" || local result="$?" + [[ result -eq 0 ]] || fail "Error adding object one" + chmod 000 "$LOCAL_FOLDER"/$bucket_name/$object_name + sleep 1 + object_is_accessible $bucket_name $object_name || local accessible=$? + [[ $accessible -eq 1 ]] || fail "Object should be inaccessible" + chmod 755 "$LOCAL_FOLDER"/$bucket_name/$object_name + sleep 1 + object_is_accessible $bucket_name $object_name || local accessible_two=$? + [[ $accessible_two -eq 0 ]] || fail "Object should be accessible" + delete_object $bucket_name/$object_name + delete_bucket $bucket_name +} + +# check info, accessiblity of bucket +@test "test_get_bucket_info" { + + local bucket_name="versity-gwtest-get-bucket-info" + + if [ -e "$LOCAL_FOLDER"/$bucket_name ]; then + chmod 755 "$LOCAL_FOLDER"/$bucket_name + sleep 1 + else + create_bucket $bucket_name || local created=$? + [[ $created -eq 0 ]] || fail "Error creating bucket" + fi + chmod 000 "$LOCAL_FOLDER"/$bucket_name + sleep 1 + bucket_is_accessible $bucket_name || local accessible=$? + [[ $accessible -eq 1 ]] || fail "Bucket should be inaccessible" + chmod 755 "$LOCAL_FOLDER"/$bucket_name + sleep 1 + bucket_is_accessible $bucket_name || local accessible_two=$? + [[ $accessible_two -eq 0 ]] || fail "Bucket should be accessible" + delete_bucket $bucket_name +} diff --git a/tests/s3_bucket_tests.sh b/tests/s3_bucket_tests.sh index 39b4cf6..50f185a 100755 --- a/tests/s3_bucket_tests.sh +++ b/tests/s3_bucket_tests.sh @@ -1,221 +1,7 @@ #!/usr/bin/env bats -source ./tests/tests.sh - -# create an AWS bucket -# param: bucket name -# return 0 for success, 1 for failure -create_bucket() { - if [ $# -ne 1 ]; then - echo "create bucket missing bucket name" - return 1 - fi - local exit_code=0 - local error - error=$(aws s3 mb s3://"$1" 2>&1) || exit_code=$? - if [ $exit_code -ne 0 ]; then - echo "error creating bucket: $error" - return 1 - fi - return 0 -} - -# delete an AWS bucket -# param: bucket name -# return 0 for success, 1 for failure -delete_bucket() { - if [ $# -ne 1 ]; then - echo "delete bucket missing bucket name" - return 1 - fi - local exit_code=0 - local error - error=$(aws s3 rb s3://"$1" 2>&1) || exit_code="$?" - if [ $exit_code -ne 0 ]; then - if [[ "$error" == *"The specified bucket does not exist"* ]]; then - return 0 - else - echo "error deleting bucket: $error" - return 1 - fi - fi - return 0 -} - -# check if bucket exists -# param: bucket name -# return 0 for true, 1 for false, 2 for error -bucket_exists() { - if [ $# -ne 1 ]; then - echo "bucket exists check missing bucket name" - return 2 - fi - local exit_code=0 - local error - error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?" - echo "Exit code: $exit_code, error: $error" - if [ $exit_code -ne 0 ]; then - if [[ "$error" == *"The specified bucket does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then - return 1 - else - echo "error checking if bucket exists: $error" - return 2 - fi - fi - return 0 -} - -# create bucket if it doesn't exist -# param: bucket name -# return 0 for success, 1 for failure -check_and_create_bucket() { - if [ $# -ne 1 ]; then - echo "bucket creation function requires bucket name" - return 1 - fi - local exists_result - bucket_exists "$1" || exists_result=$? - if [[ $exists_result -eq 2 ]]; then - echo "Bucket existence check error" - return 1 - fi - local create_result - if [[ $exists_result -eq 1 ]]; then - create_bucket "$1" || create_result=$? - if [[ $create_result -ne 0 ]]; then - echo "Error creating bucket" - return 1 - fi - fi - return 0 -} - -# check if object exists on S3 via gateway -# param: object path -# return 0 for true, 1 for false, 2 for error -object_exists() { - if [ $# -ne 1 ]; then - echo "object exists check missing object name" - return 2 - fi - local exit_code=0 - local error - error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?" - if [ $exit_code -ne 0 ]; then - if [[ "$error" == "" ]]; then - return 1 - else - echo "error checking if object exists: $error" - return 2 - fi - fi - return 0 -} - -# add object to versitygw -# params: source file, destination copy location -# return 0 for success, 1 for failure -put_object() { - if [ $# -ne 2 ]; then - echo "put object command requires source, destination" - return 1 - fi - local exit_code=0 - local error - error=$(aws s3 cp "$1" s3://"$2" 2>&1) || exit_code=$? - if [ $exit_code -ne 0 ]; then - echo "error copying object to bucket: $error" - return 1 - fi - return 0 -} - -# add object to versitygw if it doesn't exist -# params: source file, destination copy location -# return 0 for success or already exists, 1 for failure -check_and_put_object() { - if [ $# -ne 2 ]; then - echo "check and put object function requires source, destination" - return 1 - fi - object_exists "$2" || local exists_result=$? - if [ $exists_result -eq 2 ]; then - echo "error checking if object exists" - return 1 - fi - if [ $exists_result -eq 1 ]; then - put_object "$1" "$2" || local put_result=$? - if [ $put_result -ne 0 ]; then - echo "error adding object" - return 1 - fi - fi - return 0 -} - -# delete object from versitygw -# param: object location -# return 0 for success, 1 for failure -delete_object() { - if [ $# -ne 1 ]; then - echo "delete object command requires object parameter" - return 1 - fi - local exit_code=0 - local error - error=$(aws s3 rm s3://"$1" 2>&1) || exit_code=$? - if [ $exit_code -ne 0 ]; then - echo "error deleting object: $error" - return 1 - fi - return 0 -} - -# list buckets on versitygw -# no params -# export bucket_array (bucket names) on success, return 1 for failure -list_buckets() { - local exit_code=0 - local output - output=$(aws s3 ls 2>&1) || exit_code=$? - if [ $exit_code -ne 0 ]; then - echo "error listing buckets: $output" - return 1 - fi - - bucket_array=() - while IFS= read -r line; do - bucket_name=$(echo "$line" | awk '{print $NF}') - bucket_array+=("$bucket_name") - done <<< "$output" - - export bucket_array -} - -# list objects on versitygw, in bucket or folder -# param: path of bucket or folder -# export object_array (object names) on success, return 1 for failure -list_objects() { - if [ $# -ne 1 ]; then - echo "list objects command requires bucket or folder" - return 1 - fi - local exit_code=0 - local output - output=$(aws s3 ls s3://"$1" 2>&1) || exit_code=$? - if [ $exit_code -ne 0 ]; then - echo "error listing objects: $output" - return 1 - fi - - object_array=() - while IFS= read -r line; do - object_name=$(echo "$line" | awk '{print $NF}') - object_array+=("$object_name") - done <<< "$output" - - export object_array -} +source ./tests/setup.sh +source ./tests/util.sh # test creation and deletion of bucket on versitygw @test "create_delete_bucket_test" { @@ -320,7 +106,7 @@ list_objects() { } # test listing a bucket's objects on versitygw -@test test_list_objects { +@test "test_list_objects" { bucket_name="versity-gwtest-list-object" object_one="test-file-one" @@ -351,3 +137,48 @@ list_objects() { delete_bucket $bucket_name rm $object_one $object_two } + +# test ability to retrieve bucket ACLs +@test "test_get_bucket_acl" { + + local bucket_name="versity-gwtest-get-bucket-acl" + check_and_create_bucket $bucket_name || local created=$? + [[ $created -eq 0 ]] || fail "Error creating bucket" + get_bucket_acl $bucket_name || local result=$? + [[ $result -eq 0 ]] || fail "Error retrieving acl" + id=$(echo "$acl" | jq '.Owner.ID') + [[ $id == '"'"$AWS_ACCESS_KEY_ID"'"' ]] || fail "Acl mismatch" + delete_bucket $bucket_name +} + +# test ability to delete multiple objects from bucket +@test "test_delete_objects" { + + local bucket_name="versity-gwtest-delete-objects" + local object_one="test-file-one" + local object_two="test-file-two" + + touch "$object_one" "$object_two" + check_and_create_bucket $bucket_name || local result_one=$? + [[ $result_one -eq 0 ]] || fail "Error creating bucket" + put_object "$object_one" "$bucket_name"/"$object_one" || local result_two=$? + [[ $result_two -eq 0 ]] || fail "Error adding object one" + put_object "$object_two" "$bucket_name"/"$object_two" || local result_three=$? + [[ $result_three -eq 0 ]] || fail "Error adding object two" + + error=$(aws s3api delete-objects --bucket $bucket_name --delete '{ + "Objects": [ + {"Key": "test-file-one"}, + {"Key": "test-file-two"} + ] + }') || local result=$? + [[ $result -eq 0 ]] || fail "Error deleting objects: $error" + + object_exists "$bucket_name"/"$object_one" || local exists_one=$? + [[ $exists_one -eq 1 ]] || fail "Object one not deleted" + object_exists "$bucket_name"/"$object_two" || local exists_two=$? + [[ $exists_two -eq 1 ]] || fail "Object two not deleted" + + delete_bucket $bucket_name + rm "$object_one" "$object_two" +} diff --git a/tests/tests.sh b/tests/setup.sh similarity index 100% rename from tests/tests.sh rename to tests/setup.sh diff --git a/tests/util.sh b/tests/util.sh new file mode 100644 index 0000000..463c296 --- /dev/null +++ b/tests/util.sh @@ -0,0 +1,273 @@ +# create an AWS bucket +# param: bucket name +# return 0 for success, 1 for failure +create_bucket() { + if [ $# -ne 1 ]; then + echo "create bucket missing bucket name" + return 1 + fi + local exit_code=0 + local error + error=$(aws s3 mb s3://"$1" 2>&1) || exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "error creating bucket: $error" + return 1 + fi + return 0 +} + +# delete an AWS bucket +# param: bucket name +# return 0 for success, 1 for failure +delete_bucket() { + if [ $# -ne 1 ]; then + echo "delete bucket missing bucket name" + return 1 + fi + local exit_code=0 + local error + error=$(aws s3 rb s3://"$1" 2>&1) || exit_code="$?" + if [ $exit_code -ne 0 ]; then + if [[ "$error" == *"The specified bucket does not exist"* ]]; then + return 0 + else + echo "error deleting bucket: $error" + return 1 + fi + fi + return 0 +} + +# check if bucket exists +# param: bucket name +# return 0 for true, 1 for false, 2 for error +bucket_exists() { + if [ $# -ne 1 ]; then + echo "bucket exists check missing bucket name" + return 2 + fi + local exit_code=0 + local error + error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?" + echo "Exit code: $exit_code, error: $error" + if [ $exit_code -ne 0 ]; then + if [[ "$error" == *"The specified bucket does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then + return 1 + else + echo "error checking if bucket exists: $error" + return 2 + fi + fi + return 0 +} + +# create bucket if it doesn't exist +# param: bucket name +# return 0 for success, 1 for failure +check_and_create_bucket() { + if [ $# -ne 1 ]; then + echo "bucket creation function requires bucket name" + return 1 + fi + local exists_result + bucket_exists "$1" || exists_result=$? + if [[ $exists_result -eq 2 ]]; then + echo "Bucket existence check error" + return 1 + fi + local create_result + if [[ $exists_result -eq 1 ]]; then + create_bucket "$1" || create_result=$? + if [[ $create_result -ne 0 ]]; then + echo "Error creating bucket" + return 1 + fi + fi + return 0 +} + +# check if object exists on S3 via gateway +# param: object path +# return 0 for true, 1 for false, 2 for error +object_exists() { + if [ $# -ne 1 ]; then + echo "object exists check missing object name" + return 2 + fi + local exit_code=0 + local error + error=$(aws s3 ls s3://"$1" 2>&1) || exit_code="$?" + if [ $exit_code -ne 0 ]; then + if [[ "$error" == "" ]]; then + return 1 + else + echo "error checking if object exists: $error" + return 2 + fi + fi + return 0 +} + +# add object to versitygw +# params: source file, destination copy location +# return 0 for success, 1 for failure +put_object() { + if [ $# -ne 2 ]; then + echo "put object command requires source, destination" + return 1 + fi + local exit_code=0 + local error + error=$(aws s3 cp "$1" s3://"$2" 2>&1) || exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "error copying object to bucket: $error" + return 1 + fi + return 0 +} + +# add object to versitygw if it doesn't exist +# params: source file, destination copy location +# return 0 for success or already exists, 1 for failure +check_and_put_object() { + if [ $# -ne 2 ]; then + echo "check and put object function requires source, destination" + return 1 + fi + object_exists "$2" || local exists_result=$? + if [ $exists_result -eq 2 ]; then + echo "error checking if object exists" + return 1 + fi + if [ $exists_result -eq 1 ]; then + put_object "$1" "$2" || local put_result=$? + if [ $put_result -ne 0 ]; then + echo "error adding object" + return 1 + fi + fi + return 0 +} + +# delete object from versitygw +# param: object location +# return 0 for success, 1 for failure +delete_object() { + if [ $# -ne 1 ]; then + echo "delete object command requires object parameter" + return 1 + fi + local exit_code=0 + local error + error=$(aws s3 rm s3://"$1" 2>&1) || exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "error deleting object: $error" + return 1 + fi + return 0 +} + +# list buckets on versitygw +# no params +# export bucket_array (bucket names) on success, return 1 for failure +list_buckets() { + local exit_code=0 + local output + output=$(aws s3 ls 2>&1) || exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "error listing buckets: $output" + return 1 + fi + + bucket_array=() + while IFS= read -r line; do + bucket_name=$(echo "$line" | awk '{print $NF}') + bucket_array+=("$bucket_name") + done <<< "$output" + + export bucket_array +} + +# list objects on versitygw, in bucket or folder +# param: path of bucket or folder +# export object_array (object names) on success, return 1 for failure +list_objects() { + if [ $# -ne 1 ]; then + echo "list objects command requires bucket or folder" + return 1 + fi + local exit_code=0 + local output + output=$(aws s3 ls s3://"$1" 2>&1) || exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "error listing objects: $output" + return 1 + fi + + object_array=() + while IFS= read -r line; do + object_name=$(echo "$line" | awk '{print $NF}') + object_array+=("$object_name") + done <<< "$output" + + export object_array +} + +# check if bucket info can be retrieved +# param: path of bucket or folder +# return 0 for yes, 1 for no, 2 for error +bucket_is_accessible() { + if [ $# -ne 1 ]; then + echo "bucket accessibility check missing bucket name" + return 2 + fi + local exit_code=0 + local error + error=$(aws s3api head-bucket --bucket "$1" 2>&1) || exit_code="$?" + if [ $exit_code -eq 0 ]; then + return 0 + fi + if [[ "$error" == *"500"* ]]; then + return 1 + fi + echo "Error checking bucket accessibility: $error" + return 2 +} + +# check if object info (etag) is accessible +# param: path of object +# return 0 for yes, 1 for no, 2 for error +object_is_accessible() { + if [ $# -ne 2 ]; then + echo "object accessibility check missing bucket and/or key" + return 2 + fi + local exit_code=0 + object_data=$(aws s3api head-object --bucket "$1" --key "$2" 2>&1) || exit_code="$?" + if [ $exit_code -ne 0 ]; then + echo "Error obtaining object data: $object_data" + return 2 + fi + etag=$(echo "$object_data" | jq '.ETag') + if [[ "$etag" == '""' ]]; then + return 1 + fi + return 0 +} + +# get bucket acl +# param: bucket path +# export acl for success, return 1 for error +get_bucket_acl() { + if [ $# -ne 1 ]; then + echo "bucket ACL command missing bucket name" + return 1 + fi + local exit_code=0 + acl=$(aws s3api get-bucket-acl --bucket "$1" 2>&1) || exit_code="$?" + if [ $exit_code -ne 0 ]; then + echo "Error: $acl" + return 1 + fi + export acl +} diff --git a/tests/util_posix.sh b/tests/util_posix.sh new file mode 100644 index 0000000..c2450d4 --- /dev/null +++ b/tests/util_posix.sh @@ -0,0 +1,97 @@ + + +# check if object exists both on S3 and locally +# param: object path +# 0 for yes, 1 for no, 2 for error +object_exists_remote_and_local() { + if [ $# -ne 1 ]; then + echo "object existence check requires single name parameter" + return 2 + fi + object_exists "$1" || local exist_result=$? + if [[ $exist_result -eq 2 ]]; then + echo "Error checking if object exists" + return 2 + fi + if [[ $exist_result -eq 1 ]]; then + echo "Error: object doesn't exist remotely" + return 1 + fi + if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then + echo "Error: object doesn't exist locally" + return 1 + fi + return 0 +} + +# check if object doesn't exist both on S3 and locally +# param: object path +# return 0 for doesn't exist, 1 for still exists, 2 for error +object_not_exists_remote_and_local() { + if [ $# -ne 1 ]; then + echo "object non-existence check requires single name parameter" + return 2 + fi + object_exists "$1" || local exist_result=$? + if [[ $exist_result -eq 2 ]]; then + echo "Error checking if object doesn't exist" + return 2 + fi + if [[ $exist_result -eq 0 ]]; then + echo "Error: object exists remotely" + return 1 + fi + if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then + echo "Error: object exists locally" + return 1 + fi + return 0 +} + +# check if a bucket doesn't exist both on S3 and on gateway +# param: bucket name +# return: 0 for doesn't exist, 1 for does, 2 for error +bucket_not_exists_remote_and_local() { + if [ $# -ne 1 ]; then + echo "bucket existence check requires single name parameter" + return 2 + fi + bucket_exists "$1" || local exist_result=$? + if [[ $exist_result -eq 2 ]]; then + echo "Error checking if bucket exists" + return 2 + fi + if [[ $exist_result -eq 0 ]]; then + echo "Error: bucket exists remotely" + return 1 + fi + if [[ -e "$LOCAL_FOLDER"/"$1" ]]; then + echo "Error: bucket exists locally" + return 1 + fi + return 0 +} + +# check if a bucket exists both on S3 and on gateway +# param: bucket name +# return: 0 for yes, 1 for no, 2 for error +bucket_exists_remote_and_local() { + if [ $# -ne 1 ]; then + echo "bucket existence check requires single name parameter" + return 2 + fi + bucket_exists "$1" || local exist_result=$? + if [[ $exist_result -eq 2 ]]; then + echo "Error checking if bucket exists" + return 2 + fi + if [[ $exist_result -eq 1 ]]; then + echo "Error: bucket doesn't exist remotely" + return 1 + fi + if [[ ! -e "$LOCAL_FOLDER"/"$1" ]]; then + echo "Error: bucket doesn't exist locally" + return 1 + fi + return 0 +} \ No newline at end of file