diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index dcb929bc..f4312965 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -41,7 +41,6 @@ jobs: - name: Build and run run: | make testbin - echo $PATH export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn export AWS_REGION=us-east-1 diff --git a/.gitignore b/.gitignore index 9b68a4a8..49fa25a1 100644 --- a/.gitignore +++ b/.gitignore @@ -41,7 +41,7 @@ VERSION dist/ # secrets file for local github-actions testing -tests/.secrets +tests/.secrets* # IAM users files often created in testing users.json diff --git a/tests/.env.default b/tests/.env.default index 2470913f..9cc59e5c 100644 --- a/tests/.env.default +++ b/tests/.env.default @@ -1,6 +1,7 @@ AWS_PROFILE=versity AWS_ENDPOINT_URL=https://127.0.0.1:7070 VERSITY_EXE=./versitygw +RUN_VERSITYGW=true BACKEND=posix LOCAL_FOLDER=/tmp/gw BUCKET_ONE_NAME=versity-gwtest-bucket-one @@ -8,4 +9,6 @@ BUCKET_TWO_NAME=versity-gwtest-bucket-two #RECREATE_BUCKETS=true CERT=$PWD/cert.pem KEY=$PWD/versitygw.pem -S3CMD_CONFIG=./tests/s3cfg.local.default \ No newline at end of file +S3CMD_CONFIG=./tests/s3cfg.local.default +SECRETS_FILE=./tests/.secrets +MC_ALIAS=versity \ No newline at end of file diff --git a/tests/.env.static b/tests/.env.static deleted file mode 100644 index 51c965bb..00000000 --- a/tests/.env.static +++ /dev/null @@ -1,8 +0,0 @@ -AWS_PROFILE=versity -AWS_ENDPOINT_URL=http://127.0.0.1:7070 -VERSITY_EXE=./versitygw -BACKEND=posix -LOCAL_FOLDER=/tmp/gw -BUCKET_ONE_NAME=versity-gwtest-bucket-one-static -BUCKET_TWO_NAME=versity-gwtest-bucket-two-static -RECREATE_BUCKETS=false \ No newline at end of file diff --git a/tests/setup_mc.sh b/tests/setup_mc.sh index 2e4d43d5..fee7438e 100644 --- a/tests/setup_mc.sh +++ b/tests/setup_mc.sh @@ -8,7 +8,7 @@ check_for_alias() { return 2 fi while IFS= read -r line; do - if [[ $line =~ ^versity$ ]]; then + if [[ $line =~ ^"$MC_ALIAS"$ ]]; then return 0 fi done <<< "$aliases" @@ -25,7 +25,7 @@ check_add_mc_alias() { return 0 fi local set_result - error=$(mc alias set --insecure versity "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$? + error=$(mc alias set --insecure "$MC_ALIAS" "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$? if [[ $set_result -ne 0 ]]; then echo "error setting alias: $error" return 1 diff --git a/tests/test_aws.sh b/tests/test_aws.sh index a8fb37c8..6618a6b8 100755 --- a/tests/test_aws.sh +++ b/tests/test_aws.sh @@ -11,8 +11,12 @@ source ./tests/test_common.sh } # test adding and removing an object on versitygw -@test "test_put_object" { - test_common_put_object "aws" +@test "test_put_object-with-data" { + test_common_put_object_with_data "aws" +} + +@test "test_put_object-no-data" { + test_common_put_object_no_data "aws" } # test listing buckets on versitygw @@ -97,27 +101,7 @@ source ./tests/test_common.sh # test abilty to set and retrieve bucket tags @test "test-set-get-bucket-tags" { - - local key="test_key" - local value="test_value" - - setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? - [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - - get_bucket_tags "$BUCKET_ONE_NAME" || local get_result=$? - [[ $get_result -eq 0 ]] || fail "Error getting bucket tags" - tag_set=$(echo "$tags" | jq '.TagSet') - [[ $tag_set == "[]" ]] || fail "Error: tags not empty" - - put_bucket_tag "$BUCKET_ONE_NAME" $key $value - get_bucket_tags "$BUCKET_ONE_NAME" || local get_result_two=$? - [[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags" - tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key') - tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value') - [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch" - [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" + test_common_set_get_bucket_tags "aws" } # test v1 s3api list objects command @@ -184,34 +168,7 @@ source ./tests/test_common.sh # test abilty to set and retrieve object tags @test "test-set-get-object-tags" { - - local bucket_file="bucket-file" - local key="test_key" - local value="test_value" - - create_test_files "$bucket_file" || local created=$? - [[ $created -eq 0 ]] || fail "Error creating test files" - setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$? - [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" - local object_path="$BUCKET_ONE_NAME"/"$bucket_file" - put_object "aws" "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$? - [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'" - - get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result=$? - [[ $get_result -eq 0 ]] || fail "Error getting object tags" - tag_set=$(echo "$tags" | jq '.TagSet') - [[ $tag_set == "[]" ]] || fail "Error: tags not empty" - - put_object_tag "$BUCKET_ONE_NAME" $bucket_file $key $value - get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$? - [[ $get_result_two -eq 0 ]] || fail "Error getting object tags" - tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key') - tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value') - [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch" - [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch" - - delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME" - delete_test_files $bucket_file + test_common_set_get_object_tags "aws" } # test multi-part upload diff --git a/tests/test_common.sh b/tests/test_common.sh index 6fc1226b..9a1e5d22 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -25,24 +25,38 @@ test_common_create_delete_bucket() { [[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket" } -test_common_put_object() { - +test_common_put_object_with_data() { if [[ $# -ne 1 ]]; then fail "put object test requires command type" fi local object_name="test-object" + create_test_files "$object_name" || local create_result=$? + [[ $create_result -eq 0 ]] || fail "Error creating test file" + echo "test data" > "$test_file_folder"/"$object_name" +} + +test_common_put_object_no_data() { + if [[ $# -ne 1 ]]; then + fail "put object test requires command type" + fi + + local object_name="test-object" + create_test_files "$object_name" || local create_result=$? + [[ $create_result -eq 0 ]] || fail "Error creating test file" + test_common_put_object "$1" "$object_name" +} + +test_common_put_object() { + if [[ $# -ne 2 ]]; then + fail "put object test requires command type, file" + fi setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$? [[ $setup_result -eq 0 ]] || fail "error setting up bucket" - create_test_files "$object_name" || local create_result=$? - [[ $create_result -eq 0 ]] || fail "Error creating test file" - - echo "test data" > "$test_file_folder"/"$object_name" - - object="$BUCKET_ONE_NAME"/$object_name - put_object "$1" "$test_file_folder"/"$object_name" "$object" || local put_object=$? + object="$BUCKET_ONE_NAME"/"$2" + put_object "$1" "$test_file_folder"/"$2" "$object" || local put_object=$? [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket" object_exists "$1" "$object" || local exists_result_one=$? [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket" @@ -53,7 +67,7 @@ test_common_put_object() { [[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket" delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" - delete_test_files "$object_name" + delete_test_files "$2" } # common test for listing buckets @@ -95,7 +109,6 @@ test_common_list_buckets() { } test_common_list_objects() { - if [[ $# -ne 1 ]]; then echo "common test function for listing objects requires command type" return 1 @@ -133,3 +146,87 @@ test_common_list_objects() { fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})" fi } + +test_common_set_get_bucket_tags() { + + if [[ $# -ne 1 ]]; then + fail "set/get bucket tags test requires command type" + fi + + local key="test_key" + local value="test_value" + + setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? + [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" + + get_bucket_tags "$1" "$BUCKET_ONE_NAME" || local get_result=$? + [[ $get_result -eq 0 ]] || fail "Error getting bucket tags" + + if [[ $1 == 'aws' ]]; then + if [[ $tags != "" ]]; then + tag_set=$(echo "$tags" | sed '1d' | jq '.TagSet') + [[ $tag_set == "[]" ]] || fail "Error: tags not empty: $tags" + fi + else + [[ $tags == "" ]] || [[ $tags =~ "No tags found" ]] || fail "Error: tags not empty: $tags" + fi + + put_bucket_tag "$1" "$BUCKET_ONE_NAME" $key $value + get_bucket_tags "$1" "$BUCKET_ONE_NAME" || local get_result_two=$? + [[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags" + + local tag_set_key + local tag_set_value + if [[ $1 == 'aws' ]]; then + tag_set_key=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Key') + tag_set_value=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Value') + [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch" + [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch" + else + read -r tag_set_key tag_set_value <<< "$(echo "$tags" | awk 'NR==2 {print $1, $3}')" + [[ $tag_set_key == "$key" ]] || fail "Key mismatch" + [[ $tag_set_value == "$value" ]] || fail "Value mismatch" + fi + delete_bucket_tags "$1" "$BUCKET_ONE_NAME" + delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" +} + +test_common_set_get_object_tags() { + local bucket_file="bucket-file" + local key="test_key" + local value="test_value" + + create_test_files "$bucket_file" || local created=$? + [[ $created -eq 0 ]] || fail "Error creating test files" + setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$? + [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'" + local object_path="$BUCKET_ONE_NAME"/"$bucket_file" + put_object "$1" "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$? + [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'" + + get_object_tags "$1" "$BUCKET_ONE_NAME" $bucket_file || local get_result=$? + [[ $get_result -eq 0 ]] || fail "Error getting object tags" + if [[ $1 == 'aws' ]]; then + tag_set=$(echo "$tags" | sed '1d' | jq '.TagSet') + [[ $tag_set == "[]" ]] || fail "Error: tags not empty" + elif [[ ! $tags == *"No tags found"* ]]; then + fail "no tags found (tags: $tags)" + fi + + put_object_tag "$1" "$BUCKET_ONE_NAME" $bucket_file $key $value + get_object_tags "$1" "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$? + [[ $get_result_two -eq 0 ]] || fail "Error getting object tags" + if [[ $1 == 'aws' ]]; then + tag_set_key=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Key') + tag_set_value=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Value') + [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch" + [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch" + else + read -r tag_set_key tag_set_value <<< "$(echo "$tags" | awk 'NR==2 {print $1, $3}')" + [[ $tag_set_key == "$key" ]] || fail "Key mismatch" + [[ $tag_set_value == "$value" ]] || fail "Value mismatch" + fi + + delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME" + delete_test_files $bucket_file +} diff --git a/tests/test_mc.sh b/tests/test_mc.sh index b16a0ffa..2a3f216f 100755 --- a/tests/test_mc.sh +++ b/tests/test_mc.sh @@ -6,4 +6,28 @@ source ./tests/setup.sh # test mc bucket creation/deletion @test "test_create_delete_bucket_mc" { test_common_create_delete_bucket "mc" -} \ No newline at end of file +} + +@test "test_put_object-with-data-mc" { + test_common_put_object_with_data "mc" +} + +@test "test_put_object-no-data-mc" { + test_common_put_object_no_data "mc" +} + +@test "test_list_buckets_mc" { + test_common_list_buckets "mc" +} + +@test "test_list_objects_mc" { + test_common_list_objects "mc" +} + +@test "test_set_get_bucket_tags_mc" { + test_common_set_get_bucket_tags "mc" +} + +@test "test_set_get_object_tags_mc" { + test_common_set_get_object_tags "mc" +} diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index e3dda466..d0f48b11 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -10,8 +10,12 @@ source ./tests/util.sh } # test s3cmd put object -@test "test_put_object_s3cmd" { - test_common_put_object "s3cmd" +@test "test_put_object_with_data_s3cmd" { + test_common_put_object_with_data "s3cmd" +} + +@test "test_put_object_no_data_s3cmd" { + test_common_put_object_no_data "s3cmd" } # test listing buckets on versitygw diff --git a/tests/util.sh b/tests/util.sh index 8b9e8598..9ad7ed20 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -18,7 +18,7 @@ create_bucket() { elif [[ $1 == "s3cmd" ]]; then error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == "mc" ]]; then - error=$(mc --insecure mb versity/"$2" 2>&1) || exit_code=$? + error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? else echo "invalid command type $1" return 1 @@ -34,14 +34,21 @@ create_bucket() { # param: bucket name # return 0 for success, 1 for failure delete_bucket() { - if [ $# -ne 1 ]; then - echo "delete bucket missing bucket name" + if [ $# -ne 2 ]; then + echo "delete bucket missing command type, bucket name" return 1 fi local exit_code=0 local error - error=$(aws --no-verify-ssl s3 rb s3://"$1" 2>&1) || exit_code="$?" + if [[ $1 == 'aws' ]]; then + error=$(aws --no-verify-ssl s3 rb s3://"$2" 2>&1) || exit_code=$? + elif [[ $1 == 'mc' ]]; then + error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$? + else + echo "Invalid command type $1" + return 1 + fi if [ $exit_code -ne 0 ]; then if [[ "$error" == *"The specified bucket does not exist"* ]]; then return 0 @@ -79,7 +86,7 @@ delete_bucket_recursive() { if [[ "$error" == *"The specified bucket does not exist"* ]]; then return 0 else - echo "error deleting bucket: $error" + echo "error deleting bucket recursively: $error" return 1 fi fi @@ -91,7 +98,7 @@ delete_bucket_recursive() { # return 0 for success, 1 for failure delete_bucket_contents() { if [ $# -ne 2 ]; then - echo "delete bucket missing bucket name" + echo "delete bucket missing command id, bucket name" return 1 fi @@ -102,7 +109,7 @@ delete_bucket_contents() { elif [[ $1 == "s3cmd" ]]; then error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate del s3://"$2" --recursive --force 2>&1) || exit_code="$?" elif [[ $1 == "mc" ]]; then - error=$(mc --insecure rm --force --recursive versity/"$2" 2>&1) || exit_code="$?" + error=$(mc --insecure rm --force --recursive "$MC_ALIAS"/"$2" 2>&1) || exit_code="$?" else echo "invalid command type $1" return 1 @@ -126,18 +133,20 @@ bucket_exists() { local exit_code=0 local error if [[ $1 == 'aws' ]]; then - error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?" + error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?" + # NOTE: s3cmd sometimes takes longer with direct connection + sleep 1 + error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == 'mc' ]]; then - error=$(mc --insecure ls versity) + error=$(mc --insecure ls "$MC_ALIAS/$2" 2>&1) || exit_code=$? else echo "invalid command type: $1" return 2 fi if [ $exit_code -ne 0 ]; then - if [[ "$error" == *"The specified bucket does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then + if [[ "$error" == *"does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then return 1 else echo "error checking if bucket exists: $error" @@ -193,6 +202,7 @@ setup_bucket() { return 1 fi if [[ $RECREATE_BUCKETS == "false" ]]; then + echo "bucket data deletion success" return 0 fi fi @@ -219,11 +229,13 @@ object_exists() { return 2 fi local exit_code=0 - local error + local error="" if [[ $1 == 'aws' ]]; then error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?" elif [[ $1 == 's3cmd' ]]; then error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?" + elif [[ $1 == 'mc' ]]; then + error=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? else echo "invalid command type $1" return 2 @@ -235,8 +247,8 @@ object_exists() { echo "error checking if object exists: $error" return 2 fi - # s3cmd returns empty when object doesn't exist, rather than error - elif [[ $1 == 's3cmd' ]] && [[ $error == "" ]]; then + # s3cmd, mc return empty when object doesn't exist, rather than error + elif [[ ( $1 == 's3cmd' ) || ( $1 == 'mc' ) ]] && [[ $error == "" ]]; then return 1 fi return 0 @@ -255,8 +267,9 @@ put_object() { if [[ $1 == 'aws' ]]; then error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then - echo "2: $2 3: $(dirname $3)" error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$(dirname "$3")" 2>&1) || exit_code=$? + elif [[ $1 == 'mc' ]]; then + error=$(mc --insecure cp "$2" "$MC_ALIAS"/"$(dirname "$3")" 2>&1) || exit_code=$? else echo "invalid command type $1" return 1 @@ -305,7 +318,8 @@ delete_object() { error=$(aws --no-verify-ssl s3 rm s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm s3://"$2" 2>&1) || exit_code=$? - echo "$error" + elif [[ $1 == 'mc' ]]; then + error=$(mc --insecure rm "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? else echo "invalid command type $1" return 1 @@ -322,7 +336,7 @@ delete_object() { # export bucket_array (bucket names) on success, return 1 for failure list_buckets() { if [[ $# -ne 1 ]]; then - echo "List buckets command mssing format" + echo "List buckets command missing format" return 1 fi @@ -331,7 +345,9 @@ list_buckets() { if [[ $1 == "aws" ]]; then output=$(aws --no-verify-ssl s3 ls s3:// 2>&1) || exit_code=$? elif [[ $1 == "s3cmd" ]]; then - output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://) || exit_code=$? + output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$? + elif [[ $1 == 'mc' ]]; then + output=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$? else echo "invalid format: $1" return 1 @@ -345,7 +361,7 @@ list_buckets() { bucket_array=() while IFS= read -r line; do bucket_name=$(echo "$line" | awk '{print $NF}') - bucket_array+=("$bucket_name") + bucket_array+=("${bucket_name%/}") done <<< "$output" export bucket_array @@ -365,6 +381,8 @@ list_objects() { output=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code=$? elif [[ $1 == 's3cmd' ]]; then output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code=$? + elif [[ $1 == 'mc' ]]; then + output=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || exit_code=$? else echo "invalid command type $1" return 1 @@ -463,13 +481,20 @@ get_object_acl() { # params: bucket, key, value # return: 0 for success, 1 for error put_bucket_tag() { - if [ $# -ne 3 ]; then - echo "bucket tag command missing bucket name, key, value" + if [ $# -ne 4 ]; then + echo "bucket tag command missing command type, bucket name, key, value" return 1 fi local error local result - error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]") || result=$? + if [[ $1 == 'aws' ]]; then + error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$? + elif [[ $1 == 'mc' ]]; then + error=$(mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$? + else + echo "invalid command type $1" + return 1 + fi if [[ $result -ne 0 ]]; then echo "Error adding bucket tag: $error" return 1 @@ -481,30 +506,65 @@ put_bucket_tag() { # params: bucket # export 'tags' on success, return 1 for error get_bucket_tags() { - if [ $# -ne 1 ]; then - echo "get bucket tag command missing bucket name" + if [ $# -ne 2 ]; then + echo "get bucket tag command missing command type, bucket name" return 1 fi local result - tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$1") || result=$? + if [[ $1 == 'aws' ]]; then + tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$? + elif [[ $1 == 'mc' ]]; then + tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$? + else + echo "invalid command type $1" + return 1 + fi if [[ $result -ne 0 ]]; then + if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then + export tags= + return 0 + fi echo "error getting bucket tags: $tags" return 1 fi export tags } +delete_bucket_tags() { + if [ $# -ne 2 ]; then + echo "delete bucket tag command missing command type, bucket name" + return 1 + fi + local result + if [[ $1 == 'aws' ]]; then + tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$? + elif [[ $1 == 'mc' ]]; then + tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$? + else + echo "invalid command type $1" + return 1 + fi + return 0 +} + # add tags to object # params: object, key, value # return: 0 for success, 1 for error put_object_tag() { - if [ $# -ne 4 ]; then - echo "object tag command missing object name, file, key, and/or value" + if [ $# -ne 5 ]; then + echo "object tag command missing command type, object name, file, key, and/or value" return 1 fi local error local result - error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$1" --key "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$? + if [[ $1 == 'aws' ]]; then + error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$? + elif [[ $1 == 'mc' ]]; then + error=$(mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$? + else + echo "invalid command type $1" + return 1 + fi if [[ $result -ne 0 ]]; then echo "Error adding object tag: $error" return 1 @@ -516,12 +576,19 @@ put_object_tag() { # params: bucket # export 'tags' on success, return 1 for error get_object_tags() { - if [ $# -ne 2 ]; then - echo "get object tag command missing bucket and/or key" + if [ $# -ne 3 ]; then + echo "get object tag command missing command type, bucket, and/or key" return 1 fi local result - tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$1" --key "$2") || result=$? + if [[ $1 == 'aws' ]]; then + tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$? + elif [[ $1 == 'mc' ]]; then + tags=$(mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$? + else + echo "invalid command type $1" + return 1 + fi if [[ $result -ne 0 ]]; then echo "error getting object tags: $tags" return 1 diff --git a/tests/util_mc.sh b/tests/util_mc.sh index ef734b3f..b8044818 100644 --- a/tests/util_mc.sh +++ b/tests/util_mc.sh @@ -10,12 +10,12 @@ delete_bucket_recursive_mc() { fi local exit_code=0 local error - error=$(mc --insecure rm --recursive --force versity/"$1" 2>&1) || exit_code="$?" + error=$(mc --insecure rm --recursive --force "$MC_ALIAS"/"$1" 2>&1) || exit_code="$?" if [[ $exit_code -ne 0 ]]; then echo "error deleting bucket contents: $error" return 1 fi - error=$(mc --insecure rb versity/"$1" 2>&1) || exit_code="$?" + error=$(mc --insecure rb "$MC_ALIAS"/"$1" 2>&1) || exit_code="$?" if [[ $exit_code -ne 0 ]]; then echo "error deleting bucket: $error" return 1 diff --git a/tests/versity.sh b/tests/versity.sh index 07e02fc0..dbae3593 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -7,30 +7,34 @@ check_exe_params() { elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then echo "No AWS secret access key set" return 1 - elif [ -z "$VERSITY_EXE" ]; then - echo "No versity executable location set" - return 1 - elif [ -z "$BACKEND" ]; then - echo "No backend parameter set (options: 'posix')" - return 1 elif [ -z "$AWS_PROFILE" ]; then echo "No AWS profile set" return 1 - elif [ -z "$LOCAL_FOLDER" ]; then - echo "No local storage folder set" - return 1 elif [ -z "$AWS_ENDPOINT_URL" ]; then echo "No AWS endpoint URL set" return 1 + elif [ -z "$MC_ALIAS" ]; then + echo "No mc alias set" + return 1 + elif [[ $RUN_VERSITYGW != "true" ]] && [[ $RUN_VERSITYGW != "false" ]]; then + echo "RUN_VERSITYGW must be 'true' or 'false'" + return 1 + fi + if [[ $RUN_VERSITYGW == "true" ]]; then + if [ -z "$LOCAL_FOLDER" ]; then + echo "No local storage folder set" + return 1 + elif [ -z "$VERSITY_EXE" ]; then + echo "No versity executable location set" + return 1 + elif [ -z "$BACKEND" ]; then + echo "No backend parameter set (options: 'posix')" + return 1 + fi fi } start_versity() { - if [ "$GITHUB_ACTIONS" != "true" ] && [ -r tests/.secrets ]; then - source tests/.secrets - else - echo "Warning: no secrets file found" - fi if [ -z "$VERSITYGW_TEST_ENV" ]; then if [ -r tests/.env ]; then source tests/.env @@ -41,6 +45,12 @@ start_versity() { # shellcheck source=./.env.default source "$VERSITYGW_TEST_ENV" fi + if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then + # shellcheck source=/.secrets + source "$SECRETS_FILE" + else + echo "Warning: no secrets file found" + fi check_exe_params || check_result=$? if [[ $check_result -ne 0 ]]; then @@ -48,25 +58,25 @@ start_versity() { return 1 fi - base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY $VERSITY_EXE" - if [ -n "$CERT" ] && [ -n "$KEY" ]; then - base_command+=" --cert $CERT --key $KEY" - fi - base_command+=" $BACKEND $LOCAL_FOLDER &" - eval "$base_command" + if [ "$RUN_VERSITYGW" == "true" ]; then + base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY VGW_REGION=$AWS_REGION $VERSITY_EXE" + if [ -n "$CERT" ] && [ -n "$KEY" ]; then + base_command+=" --cert $CERT --key $KEY" + fi + base_command+=" $BACKEND $LOCAL_FOLDER &" + eval "$base_command" + versitygw_pid=$! - versitygw_pid=$! - export versitygw_pid \ - AWS_ACCESS_KEY_ID \ - AWS_SECRET_ACCESS_KEY \ - VERSITY_EXE \ - BACKEND \ - AWS_PROFILE \ - LOCAL_FOLDER \ - AWS_ENDPOINT_URL + export versitygw_pid VERSITY_EXE BACKEND LOCAL_FOLDER + fi + + export versitygw_pid AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL } stop_versity() { + if [ "$RUN_VERSITYGW" == "false" ]; then + return + fi if [ -n "$versitygw_pid" ]; then if ps -p "$versitygw_pid" > /dev/null; then kill "$versitygw_pid"