diff --git a/tests/commands/delete_bucket.sh b/tests/commands/delete_bucket.sh index 55007ff..bdf80d2 100644 --- a/tests/commands/delete_bucket.sh +++ b/tests/commands/delete_bucket.sh @@ -62,4 +62,17 @@ delete_bucket_rest() { return 1 fi return 0 +} + +delete_bucket_rest_expect_error() { + if ! check_param_count_gt "bucket, env vars, response code, error code, message" 5 $#; then + return 1 + fi + env_vars="BUCKET_NAME=$1 $2" + if ! send_rest_command_expect_error "$env_vars" "./tests/rest_scripts/delete_bucket.sh" "$3" "$4" "$5"; then + log 2 "error sending delete bucket command or error mismatch" + return 1 + fi + return 0 + } \ No newline at end of file diff --git a/tests/commands/put_bucket_acl.sh b/tests/commands/put_bucket_acl.sh index f21415f..365097c 100644 --- a/tests/commands/put_bucket_acl.sh +++ b/tests/commands/put_bucket_acl.sh @@ -16,6 +16,7 @@ source ./tests/util/util_file.sh source ./tests/commands/command.sh +source ./tests/drivers/put_bucket_acl/put_bucket_acl.sh source ./tests/drivers/rest.sh put_bucket_acl_s3api() { diff --git a/tests/drivers/create_bucket/create_bucket_rest.sh b/tests/drivers/create_bucket/create_bucket_rest.sh index 312df35..8060c98 100644 --- a/tests/drivers/create_bucket/create_bucket_rest.sh +++ b/tests/drivers/create_bucket/create_bucket_rest.sh @@ -18,6 +18,7 @@ source ./tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh source ./tests/drivers/get_object/get_object_rest.sh source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh source ./tests/drivers/put_object/put_object_rest.sh +source ./tests/drivers/user.sh setup_and_create_bucket_and_check_acl() { if [ "$RECREATE_BUCKETS" == "false" ]; then diff --git a/tests/drivers/delete_object/delete_object_rest.sh b/tests/drivers/delete_object/delete_object_rest.sh new file mode 100644 index 0000000..bcac729 --- /dev/null +++ b/tests/drivers/delete_object/delete_object_rest.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/commands/list_objects_v2.sh +source ./tests/drivers/xml.sh +source ./tests/util/util_legal_hold.sh + +list_and_delete_objects() { + log 6 "list_and_delete_objects" + if ! check_param_count "list_and_delete_objects" "bucket" 1 $#; then + return 1 + fi + if ! list_objects_rest "$1" "parse_objects_list_rest"; then + log 2 "error getting object list" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "objects: ${object_array[*]}" + for object in "${object_array[@]}"; do + if ! clear_object_in_bucket "$1" "$object"; then + log 2 "error deleting object $object" + return 1 + fi + done + + if ! delete_old_versions "$1"; then + log 2 "error deleting old version" + return 1 + fi + return 0 +} + +delete_old_versions() { + if ! check_param_count "delete_old_versions" "bucket" 1 $#; then + return 1 + fi + if ! list_object_versions "rest" "$1"; then + log 2 "error listing object versions" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "versions: $versions" + version_keys=() + version_ids=() + + if ! parse_version_data_by_type "rest" "$2"; then + log 2 "error parsing version data" + return 1 + fi + + log 5 "version keys: ${version_keys[*]}" + log 5 "version IDs: ${version_ids[*]}" + for idx in "${!version_keys[@]}"; do + if ! delete_object_version_with_or_without_retention "$1"; then + log 2 "error deleting version with or without retention" + return 1 + fi + done +} + +delete_object_version_with_or_without_retention() { + if ! check_param_count "delete_object_version_with_or_without_retention" "bucket" 1 $#; then + return 1 + fi + log 5 "idx: $idx" + log 5 "version ID: ${version_ids[$idx]}" + # shellcheck disable=SC2154 + if [ "$lock_config_exists" == "true" ]; then + if ! check_remove_legal_hold_versions "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + log 2 "error checking, removing legal hold versions" + fi + if ! delete_object_version_rest_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + log 2 "error deleting object version, bypassing retention" + return 1 + fi + else + if ! delete_object_version_rest "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then + log 2 "error deleting object version" + return 1 + fi + fi + log 5 "successfully deleted version with key '${version_keys[$idx]}', id '${version_ids[$idx]}'" + return 0 +} diff --git a/tests/drivers/get_bucket_acl/get_bucket_acl.sh b/tests/drivers/get_bucket_acl/get_bucket_acl.sh new file mode 100644 index 0000000..96349da --- /dev/null +++ b/tests/drivers/get_bucket_acl/get_bucket_acl.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +get_check_acl_id() { + if [ $# -ne 2 ]; then + log 2 "'get_check_acl_id' requires client, bucket" + return 1 + fi + if ! get_bucket_acl "$1" "$2"; then + log 2 "error retrieving acl" + return 1 + fi + log 5 "Initial ACLs: $acl" + if ! id=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Owner.ID' 2>&1); then + log 2 "error getting ID: $id" + return 1 + fi + if [[ $id != "$AWS_ACCESS_KEY_ID" ]]; then + # for direct, ID is canonical user ID rather than AWS_ACCESS_KEY_ID + if ! canonical_id=$(aws --no-verify-ssl s3api list-buckets --query 'Owner.ID' 2>&1); then + log 2 "error getting canonical ID: $canonical_id" + return 1 + fi + canonical_id="$(echo -n "$canonical_id" | grep -v "InsecureRequestWarning" | sed "s/\"//g")" + log 5 "canonical ID: $canonical_id" + if [[ $id != "$canonical_id" ]]; then + log 2 "acl ID doesn't match AWS key or canonical ID" + return 1 + fi + fi + return 0 +} + +get_check_acl_after_first_put() { + if [ $# -ne 2 ]; then + log 2 "'get_check_acl_after_first_put' requires client, bucket" + return 1 + fi + if ! get_bucket_acl "$1" "$BUCKET_ONE_NAME"; then + log 2 "error retrieving second ACL" + return 1 + fi + log 5 "Acls after 1st put: $acl" + if ! public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants[1]' 2>&1); then + log 2 "error getting public grants: $public_grants" + return 1 + fi + if ! permission=$(echo "$public_grants" | jq -r '.Permission' 2>&1); then + log 2 "error getting permission: $permission" + return 1 + fi + if [[ $permission != "READ" ]]; then + log 2 "incorrect permission ($permission)" + return 1 + fi + return 0 +} + +get_check_acl_after_second_put() { + if [ $# -ne 2 ]; then + log 2 "'get_check_acl_after_second_put' requires client, bucket" + return 1 + fi + if ! get_bucket_acl "$1" "$BUCKET_ONE_NAME"; then + log 2 "error retrieving third ACL" + return 1 + fi + if ! public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants' 2>&1); then + log 2 "error retrieving public grants: $public_grants" + return 1 + fi + if ! public_grant_length=$(echo "$public_grants" | jq -r 'length' 2>&1); then + log 2 "Error retrieving public grant length: $public_grant_length" + return 1 + fi + if [[ $public_grant_length -ne 2 ]]; then + log 2 "incorrect grant length for private ACL ($public_grant_length)" + return 1 + fi + if ! permission=$(echo "$public_grants" | jq -r '.[0].Permission' 2>&1); then + log 2 "Error retrieving permission: $permission" + return 1 + fi + if [[ $permission != "FULL_CONTROL" ]]; then + log 2 "incorrect permission ($permission)" + return 1 + fi + return 0 +} + +check_direct_display_name() { + if ! display_name=$(echo "$owner" | xmllint --xpath '//*[local-name()="DisplayName"]/text()' - 2>&1); then + log 2 "error getting display name: $display_name" + return 1 + fi + if [ "$display_name" != "$DIRECT_DISPLAY_NAME" ]; then + log 2 "display name mismatch (expected '$DIRECT_DISPLAY_NAME', actual '$display_name')" + return 1 + fi +} diff --git a/tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh b/tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh index 8e1d9a4..d6b3977 100644 --- a/tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh +++ b/tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh @@ -73,3 +73,46 @@ check_for_display_name() { fi return 0 } + +check_acl_rest() { + if ! check_param_count_v2 "acl file" 1 $#; then + return 1 + fi + log 5 "acl: $(cat "$1")" + if ! access_control_policy=$(xmllint --xpath '//*[local-name()="AccessControlPolicy"]' "$1" 2>&1); then + log 2 "error getting access control policy: $access_control_policy" + return 1 + fi + if ! owner=$(echo "$access_control_policy" | xmllint --xpath '//*[local-name()="Owner"]' - 2>&1); then + log 2 "error getting owner information: $owner" + return 1 + fi + if [ "$DIRECT" == "true" ]; then + if ! check_direct_display_name; then + log 2 "error checking direct display name" + return 1 + fi + else + if ! id=$(echo "$owner" | xmllint --xpath '//*[local-name()="ID"]/text()' - 2>&1); then + log 2 "error getting ID: $id" + return 1 + fi + if [ "$id" != "$AWS_ACCESS_KEY_ID" ]; then + log 2 "ID mismatch" + return 1 + fi + fi + return 0 +} + +get_and_check_acl_rest() { + if [ $# -ne 1 ]; then + log 2 "'get_and_check_acl_rest' requires bucket name" + return 1 + fi + if ! get_bucket_acl_rest "$1" "" "check_acl_rest"; then + log 2 "error getting and checking acl" + return 1 + fi + return 0 +} diff --git a/tests/drivers/get_bucket_acl/get_bucket_acl_s3api.sh b/tests/drivers/get_bucket_acl/get_bucket_acl_s3api.sh new file mode 100644 index 0000000..eb52162 --- /dev/null +++ b/tests/drivers/get_bucket_acl/get_bucket_acl_s3api.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +get_check_acl_after_policy() { + if [ $# -ne 1 ]; then + log 2 "'get_check_acl_after_policy' requires bucket name" + return 1 + fi + if ! get_bucket_acl "s3api" "$1"; then + log 2 "error getting bucket acl" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "ACL: $acl" + if ! second_grant=$(echo "$acl" | jq -r ".Grants[1]" 2>&1); then + log 2 "error getting second grant: $second_grant" + return 1 + fi + if ! second_grantee=$(echo "$second_grant" | jq -r ".Grantee" 2>&1); then + log 2 "error getting second grantee: $second_grantee" + return 1 + fi + if ! permission=$(echo "$second_grant" | jq -r ".Permission" 2>&1); then + log 2 "error getting permission: $permission" + return 1 + fi + log 5 "second grantee: $second_grantee" + if [[ $permission != "READ" ]]; then + log 2 "incorrect permission: $permission" + return 1 + fi + if [[ $DIRECT == "true" ]]; then + if ! uri=$(echo "$second_grantee" | jq -r ".URI" 2>&1); then + log 2 "error getting uri: $uri" + return 1 + fi + if [[ $uri != "http://acs.amazonaws.com/groups/global/AllUsers" ]]; then + log 2 "unexpected URI: $uri" + return 1 + fi + else + if ! id=$(echo "$second_grantee" | jq -r ".ID" 2>&1); then + log 2 "error getting ID: $id" + return 1 + fi + if [[ $id != "all-users" ]]; then + log 2 "unexpected ID: $id" + return 1 + fi + fi +} diff --git a/tests/drivers/get_bucket_acl/get_bucket_acl_s3cmd.sh b/tests/drivers/get_bucket_acl/get_bucket_acl_s3cmd.sh new file mode 100644 index 0000000..37efce5 --- /dev/null +++ b/tests/drivers/get_bucket_acl/get_bucket_acl_s3cmd.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +get_check_default_acl_s3cmd() { + if [ $# -ne 1 ]; then + log 2 "'get_check_acl_s3cmd' requires bucket name" + return 1 + fi + if ! get_bucket_acl "s3cmd" "$BUCKET_ONE_NAME"; then + log 2 "error retrieving acl" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "Initial ACLs: $acl" + acl_line=$(echo "$acl" | grep "ACL") + user_id=$(echo "$acl_line" | awk '{print $2}') + if [[ $DIRECT == "true" ]]; then + if [[ $user_id != "$DIRECT_DISPLAY_NAME:" ]]; then + log 2 "ID mismatch ($user_id, $DIRECT_DISPLAY_NAME)" + return 1 + fi + else + if [[ $user_id != "$AWS_ACCESS_KEY_ID:" ]]; then + log 2 "ID mismatch ($user_id, $AWS_ACCESS_KEY_ID)" + return 1 + fi + fi + permission=$(echo "$acl_line" | awk '{print $3}') + if [[ $permission != "FULL_CONTROL" ]]; then + log 2 "Permission mismatch ($permission)" + return 1 + fi + return 0 +} + +get_check_post_change_acl_s3cmd() { + if [ $# -ne 1 ]; then + log 2 "'get_check_post_change_acl_s3cmd' requires bucket name" + return 1 + fi + if ! get_bucket_acl "s3cmd" "$1"; then + log 2 "error retrieving acl" + return 1 + fi + log 5 "ACL after read put: $acl" + acl_lines=$(echo "$acl" | grep "ACL") + log 5 "ACL lines: $acl_lines" + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "$acl_lines" + log 5 "lines: ${lines[*]}" + if [[ ${#lines[@]} -ne 2 ]]; then + log 2 "unexpected number of ACL lines: ${#lines[@]}" + return 1 + fi + anon_name=$(echo "${lines[1]}" | awk '{print $2}') + anon_permission=$(echo "${lines[1]}" | awk '{print $3}') + if [[ $anon_name != "*anon*:" ]]; then + log 2 "unexpected anon name: $anon_name" + return 1 + fi + if [[ $anon_permission != "READ" ]]; then + log 2 "unexpected anon permission: $anon_permission" + return 1 + fi + return 0 +} diff --git a/tests/drivers/list_objects/list_objects_rest.sh b/tests/drivers/list_objects/list_objects_rest.sh index e106b79..5563fa7 100644 --- a/tests/drivers/list_objects/list_objects_rest.sh +++ b/tests/drivers/list_objects/list_objects_rest.sh @@ -14,33 +14,6 @@ # specific language governing permissions and limitations # under the License. -# param: bucket name -# return 0 for success, 1 for failure -list_and_delete_objects() { - log 6 "list_and_delete_objects" - if ! check_param_count "list_and_delete_objects" "bucket" 1 $#; then - return 1 - fi - if ! list_objects_rest "$1" "parse_objects_list_rest"; then - log 2 "error getting object list" - return 1 - fi - # shellcheck disable=SC2154 - log 5 "objects: ${object_array[*]}" - for object in "${object_array[@]}"; do - if ! clear_object_in_bucket "$1" "$object"; then - log 2 "error deleting object $object" - return 1 - fi - done - - if ! delete_old_versions "$1"; then - log 2 "error deleting old version" - return 1 - fi - return 0 -} - list_check_single_object() { if ! check_param_count_gt "bucket, key, env params (optional)" 2 $#; then return 1 diff --git a/tests/drivers/openssl.sh b/tests/drivers/openssl.sh index 36e42cd..30de786 100644 --- a/tests/drivers/openssl.sh +++ b/tests/drivers/openssl.sh @@ -14,7 +14,7 @@ # specific language governing permissions and limitations # under the License. -source ./tests/util/util_xml.sh +source ./tests/drivers/xml.sh send_via_openssl() { if ! check_param_count_v2 "command file" 1 $#; then diff --git a/tests/drivers/put_bucket_acl/put_bucket_acl.sh b/tests/drivers/put_bucket_acl/put_bucket_acl.sh new file mode 100644 index 0000000..9f0d3b8 --- /dev/null +++ b/tests/drivers/put_bucket_acl/put_bucket_acl.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +setup_acl() { + if ! check_param_count "setup_acl" "acl file, grantee type, grantee, permission, owner ID" 5 $#; then + return 1 + fi + cat < "$1" + + + $5 + + + + + $3 + + $4 + + + +EOF +} + +setup_acl_json() { + if [ $# -ne 5 ]; then + log 2 "'setup_acl_json' requires acl file, grantee type, grantee ID, permission, owner ID" + return 1 + fi + cat < "$1" +{ + "Grants": [ + { + "Grantee": { + "Type": "$2", + "ID": "$3" + }, + "Permission": "$4" + } + ], + "Owner": { + "ID": "$5" + } +} +EOF +} diff --git a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh index b9d49ab..52585dc 100644 --- a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh +++ b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh @@ -31,3 +31,45 @@ put_bucket_acl_success_or_access_denied() { fi return 0 } + +put_invalid_acl_rest_verify_failure() { + if [ $# -ne 2 ]; then + log 2 "'put_invalid_acl_rest_verify_failure' requires bucket name, ACL file" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" ACL_FILE="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_bucket_acl.sh); then + log 2 "error attempting to put bucket acl: $result" + return 1 + fi + if [ "$result" != "400" ]; then + log 2 "response returned code: $result (error: $(cat "$TEST_FILE_FOLDER/response.txt"))" + return 1 + fi + if ! error_code=$(xmllint --xpath '//*[local-name()="Code"]/text()' "$TEST_FILE_FOLDER/response.txt" 2>&1); then + log 2 "error getting display name: $error_code" + return 1 + fi + if [ "$error_code" != "MalformedACLError" ]; then + log 2 "invalid error code, expected 'MalformedACLError', was '$error_code'" + return 1 + fi + return 0 +} + +# param: bucket name +# return 0 for success, 1 for failure +check_ownership_rule_and_reset_acl() { + if [ $# -ne 1 ]; then + log 2 "'check_ownership_rule_and_reset_acl' requires bucket name" + return 1 + fi + if ! object_ownership_rule=$(get_bucket_ownership_controls_rest "$1" 2>&1); then + log 2 "error getting bucket ownership controls" + return 1 + fi + log 5 "ownership rule: $object_ownership_rule" + if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then + log 2 "error resetting bucket ACL" + return 1 + fi +} diff --git a/tests/drivers/put_object/put_object_rest.sh b/tests/drivers/put_object/put_object_rest.sh index e644fe6..03f8a6f 100644 --- a/tests/drivers/put_object/put_object_rest.sh +++ b/tests/drivers/put_object/put_object_rest.sh @@ -34,4 +34,23 @@ put_object_success_or_access_denied() { fi fi return 0 +} + +setup_bucket_and_add_file() { + if ! check_param_count_v2 "bucket, filename" 2 $#; then + return 1 + fi + if ! setup_bucket "$1"; then + log 2 "error setting up bucket" + return 1 + fi + if ! create_test_files "$2"; then + log 2 "error creating test file" + return 1 + fi + if ! put_object_rest "$TEST_FILE_FOLDER/$2" "$1" "$2"; then + log 2 "error putting REST object" + return 1 + fi + return 0 } \ No newline at end of file diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 0bcaf44..e70f902 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -14,7 +14,7 @@ # specific language governing permissions and limitations # under the License. -source ./tests/util/util_xml.sh +source ./tests/drivers/xml.sh check_rest_expected_error() { if ! check_param_count_v2 "response, response file, expected http code, expected error code, expected error" 5 $#; then @@ -31,6 +31,23 @@ check_rest_expected_error() { return 0 } +check_rest_go_expected_error() { + if ! check_param_count_v2 "response file, expected http code, expected error code, expected error" 4 $#; then + return 1 + fi + status_line=$(head -n 1 "$1") + status_code=$(echo "$status_line" | awk '{print $2}') + if [ "$2" != "$status_code" ]; then + log 2 "expected curl response '$2', was '$status_code'" + return 1 + fi + if ! check_xml_error_contains "$1" "$3" "$4"; then + log 2 "error checking XML error" + return 1 + fi + return 0 +} + send_rest_command() { if ! check_param_count_v2 "env vars, script" 2 $#; then return 1 @@ -146,3 +163,52 @@ if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/output.txt"; then fi return 0 } + +send_rest_go_command_expect_error() { + if [ $# -lt 3 ] && [ $(($# % 2)) -ne 1 ]; then + log 2 "'send_rest_go_command_expect_failure' param count must be 3 or greater, odd (key/value pairs)" + return 1 + fi + if ! curl_command=$(go run ./tests/rest_scripts/generate_command.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -url "$AWS_ENDPOINT_URL" "${@:4}" 2>&1); then + log 2 "error: $curl_command" + return 1 + fi + local full_command="send_command $curl_command" + log 5 "full command: $full_command" + if ! result=$(eval "${full_command[*]}" 2>&1); then + log 3 "error sending command: $result" + return 1 + fi + log 5 "result: $result" + echo -n "$result" > "$TEST_FILE_FOLDER/result.txt" + if ! check_rest_go_expected_error "$TEST_FILE_FOLDER/result.txt" "$1" "$2" "$3"; then + log 2 "error checking expected header error" + return 1 + fi + return 0 +} + +send_rest_go_command() { + if [ $# -lt 1 ] && [ $(($# % 2)) -ne 1 ]; then + log 2 "'send_rest_go_command_expect_failure' param count must be 1 or greater, odd (key/value pairs)" + return 1 + fi + if ! curl_command=$(go run ./tests/rest_scripts/generate_command.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -url "$AWS_ENDPOINT_URL" "${@:2}" 2>&1); then + log 2 "error: $curl_command" + return 1 + fi + local full_command="send_command $curl_command" + log 5 "curl command: $curl_command" + if ! result=$(eval "${full_command[*]}" 2>&1); then + log 3 "error sending command: $result" + return 1 + fi + log 5 "result: $result" + status_line=$(sed -n 1p <<< "$result") + status_code=$(echo "$status_line" | awk '{print $2}') + if [ "$1" != "$status_code" ]; then + log 2 "expected curl response '$1', was '$status_code'" + return 1 + fi + return 0 +} diff --git a/tests/drivers/user.sh b/tests/drivers/user.sh new file mode 100644 index 0000000..7c28602 --- /dev/null +++ b/tests/drivers/user.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/util/util_users.sh + +create_versitygw_acl_user_or_get_direct_user() { + if [ $# -ne 2 ]; then + log 2 "'create_versitygw_acl_user_or_get_direct_user' requires username, password" + return 1 + fi + if [ "$DIRECT" == "true" ]; then + if [ -z "$AWS_CANONICAL_ID" ] || [ -z "$ACL_AWS_CANONICAL_ID" ] || [ -z "$ACL_AWS_ACCESS_KEY_ID" ] || [ -z "$ACL_AWS_SECRET_ACCESS_KEY" ]; then + log 2 "direct ACL calls require the following env vars: AWS_CANONICAL_ID, ACL_AWS_CANONICAL_ID, ACL_AWS_ACCESS_KEY_ID, ACL_AWS_SECRET_ACCESS_KEY" + return 1 + fi + owner_canonical_id="$AWS_CANONICAL_ID" + user_canonical_id="$ACL_AWS_CANONICAL_ID" + username="$ACL_AWS_ACCESS_KEY_ID" + password="$ACL_AWS_SECRET_ACCESS_KEY" + else + owner_canonical_id="$AWS_ACCESS_KEY_ID" + if ! create_user_versitygw "$1" "$2" "user"; then + log 2 "error creating versitygw user" + return 1 + fi + # shellcheck disable=SC2154 + user_canonical_id="$1" + username="$1" + # shellcheck disable=SC2154 + password="$2" + fi + echo "$owner_canonical_id" + echo "$user_canonical_id" + echo "$username" + echo "$password" +} diff --git a/tests/util/util_xml.sh b/tests/drivers/xml.sh similarity index 100% rename from tests/util/util_xml.sh rename to tests/drivers/xml.sh diff --git a/tests/rest_scripts/generate_command.go b/tests/rest_scripts/generate_command.go index 0928aac..ecba383 100644 --- a/tests/rest_scripts/generate_command.go +++ b/tests/rest_scripts/generate_command.go @@ -7,6 +7,7 @@ import ( "flag" "fmt" "log" + "sort" "strings" "time" ) @@ -21,6 +22,7 @@ var awsAccessKeyId *string var awsSecretAccessKey *string var serviceName *string var debug *bool +var signedParamsMap restParams type S3Command struct { Method string @@ -32,6 +34,7 @@ type S3Command struct { AwsAccessKeyId string AwsSecretAccessKey string ServiceName string + SignedParams map[string]string currentDateTime string host string @@ -44,6 +47,27 @@ type S3Command struct { signature string } +type restParams map[string]string + +func (r *restParams) String() string { + return fmt.Sprintf("%v", *r) +} + +func (r *restParams) Set(value string) error { + *r = make(map[string]string) + pairs := strings.Split(value, ",") + for _, pair := range pairs { + kv := strings.SplitN(pair, ":", 2) + if len(kv) != 2 { + } + if len(kv) != 2 { + return fmt.Errorf("invalid key-value pair: %s", pair) + } + (*r)[kv[0]] = kv[1] + } + return nil +} + func main() { if err := checkFlags(); err != nil { log.Fatalf("Error checking flags: %v", err) @@ -59,6 +83,7 @@ func main() { AwsAccessKeyId: *awsAccessKeyId, AwsSecretAccessKey: *awsSecretAccessKey, ServiceName: *serviceName, + SignedParams: signedParamsMap, } curlShellCommand, err := s3Command.CurlShellCommand() if err != nil { @@ -78,6 +103,7 @@ func checkFlags() error { awsRegion = flag.String("awsRegion", "us-east-1", "AWS region") serviceName = flag.String("serviceName", "s3", "Service name") debug = flag.Bool("debug", false, "Print debug statements") + flag.Var(&signedParamsMap, "signedParams", "Signed params, separated by comma") // Parse the flags flag.Parse() @@ -114,6 +140,13 @@ func (s *S3Command) CurlShellCommand() (string, error) { {"x-amz-content-sha256", s.payloadHash}, {"x-amz-date", s.currentDateTime}, } + for key, value := range s.SignedParams { + s.headerValues = append(s.headerValues, []string{key, value}) + } + sort.Slice(s.headerValues, + func(i, j int) bool { + return s.headerValues[i][0] < s.headerValues[j][0] + }) s.path = "/" + s.BucketName if s.ObjectKey != "" { s.path += "/" + s.ObjectKey diff --git a/tests/test_common.sh b/tests/test_common.sh index eafcdd8..79f24ee 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -15,7 +15,6 @@ # under the License. source ./tests/setup.sh -source ./tests/util/util_acl.sh source ./tests/util/util_bucket_location.sh source ./tests/util/util_file.sh source ./tests/util/util_list_buckets.sh diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 9579bef..1aac8b2 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -30,9 +30,9 @@ source ./tests/commands/put_object.sh source ./tests/commands/put_object_retention.sh source ./tests/commands/put_object_tagging.sh source ./tests/drivers/copy_object/copy_object_rest.sh +source ./tests/drivers/xml.sh source ./tests/logger.sh source ./tests/setup.sh -source ./tests/util/util_acl.sh source ./tests/util/util_attributes.sh source ./tests/util/util_chunked_upload.sh source ./tests/util/util_delete_object.sh @@ -52,7 +52,6 @@ source ./tests/util/util_setup.sh source ./tests/util/util_tags.sh source ./tests/util/util_time.sh source ./tests/util/util_versioning.sh -source ./tests/util/util_xml.sh export RUN_USERS=true test_file="test_file" @@ -331,9 +330,6 @@ test_file="test_file" } @test "REST - PutObjectRetention - w/o request body" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1185" - fi run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME" assert_success diff --git a/tests/test_rest_acl.sh b/tests/test_rest_acl.sh index f217f06..45ed2d3 100755 --- a/tests/test_rest_acl.sh +++ b/tests/test_rest_acl.sh @@ -22,7 +22,6 @@ source ./tests/drivers/create_bucket/create_bucket_rest.sh source ./tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh source ./tests/logger.sh source ./tests/setup.sh -source ./tests/util/util_acl.sh source ./tests/util/util_object.sh source ./tests/util/util_public_access_block.sh source ./tests/util/util_setup.sh diff --git a/tests/test_rest_chunked.sh b/tests/test_rest_chunked.sh index 2de932b..fabdd8d 100755 --- a/tests/test_rest_chunked.sh +++ b/tests/test_rest_chunked.sh @@ -153,41 +153,26 @@ source ./tests/util/util_setup.sh } @test "test - REST chunked upload - sha1 trailer - invalid" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1165" - fi run chunked_upload_trailer_invalid_checksum "sha1" assert_success } @test "test - REST chunked upload - sha256 trailer - invalid" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1165" - fi run chunked_upload_trailer_invalid_checksum "sha256" assert_success } @test "test - REST chunked upload - crc32 trailer - invalid" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1165" - fi run chunked_upload_trailer_invalid_checksum "crc32" assert_success } @test "test - REST chunked upload - crc32c trailer - invalid" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1165" - fi run chunked_upload_trailer_invalid_checksum "crc32c" assert_success } @test "test - REST chunked upload - crc64nvme trailer - invalid" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1165" - fi run chunked_upload_trailer_invalid_checksum "crc64nvme" assert_success } diff --git a/tests/test_rest_delete_bucket.sh b/tests/test_rest_delete_bucket.sh new file mode 100755 index 0000000..c5c072f --- /dev/null +++ b/tests/test_rest_delete_bucket.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/setup.sh +source ./tests/drivers/delete_object/delete_object_rest.sh +source ./tests/drivers/put_object/put_object_rest.sh + +@test "REST - DeleteBucket - can delete with partial multipart upload" { + run setup_bucket "$BUCKET_ONE_NAME" + assert_success + + run create_multipart_upload_rest "$BUCKET_ONE_NAME" "test_file" "" + assert_success + + run delete_bucket_rest "$BUCKET_ONE_NAME" + assert_success +} + +@test "REST - DeleteBucket - file - non-versioning" { + run setup_bucket_and_add_file "$BUCKET_ONE_NAME" "test_file" + assert_success + + run delete_bucket_rest_expect_error "$BUCKET_ONE_NAME" "" "409" "BucketNotEmpty" "is not empty" + assert_success + + run delete_object_rest "$BUCKET_ONE_NAME" "test_file" + assert_success + + run delete_bucket_rest "$BUCKET_ONE_NAME" + assert_success +} + +@test "REST - DeleteBucket - file - versioning" { + run setup_bucket_and_add_file "$BUCKET_ONE_NAME" "test_file" + assert_success + + run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Enabled" + assert_success + + run delete_bucket_rest_expect_error "$BUCKET_ONE_NAME" "" "409" "BucketNotEmpty" "is not empty" + assert_success + + run delete_object_rest "$BUCKET_ONE_NAME" "test_file" + assert_success + + run delete_bucket_rest_expect_error "$BUCKET_ONE_NAME" "" "409" "BucketNotEmpty" "is not empty" + assert_success + + run delete_old_versions "$BUCKET_ONE_NAME" + assert_success + + run delete_bucket_rest "$BUCKET_ONE_NAME" + assert_success +} + +@test "REST - DeleteBucket - invalid x-amz-expected-bucket-owner" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1428" + fi + run setup_bucket "$BUCKET_ONE_NAME" + assert_success + + run send_rest_go_command_expect_error "400" "InvalidBucketOwnerAWSAccountID" "value of the expected bucket owner" "-method" "DELETE" "-bucketName" "$BUCKET_ONE_NAME" "-signedParams" "x-amz-expected-bucket-owner:01234567890" + assert_success +} + +@test "REST - DeleteBucket - incorrect x-amz-expected-bucket-owner" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1428" + fi + run setup_bucket "$BUCKET_ONE_NAME" + assert_success + + run send_rest_go_command_expect_error "403" "AccessDenied" "Access Denied" "-method" "DELETE" "-bucketName" "$BUCKET_ONE_NAME" "-signedParams" "x-amz-expected-bucket-owner:012345678901" + assert_success +} + +@test "REST - DeleteBucket - correct x-amz-expected-bucket-owner" { + run setup_bucket "$BUCKET_ONE_NAME" + assert_success + + run send_rest_go_command "204" "-method" "DELETE" "-bucketName" "$BUCKET_ONE_NAME" "-signedParams" "x-amz-expected-bucket-owner:$AWS_USER_ID" + assert_success +} diff --git a/tests/test_s3api_bucket.sh b/tests/test_s3api_bucket.sh index 5dffaf7..9438d31 100755 --- a/tests/test_s3api_bucket.sh +++ b/tests/test_s3api_bucket.sh @@ -50,6 +50,7 @@ source ./tests/commands/put_object_lock_configuration.sh source ./tests/commands/put_object_retention.sh source ./tests/commands/put_public_access_block.sh source ./tests/commands/select_object_content.sh +source ./tests/drivers/get_bucket_acl/get_bucket_acl.sh export RUN_USERS=true diff --git a/tests/test_s3api_policy.sh b/tests/test_s3api_policy.sh index 0bfbf60..5cfbca7 100755 --- a/tests/test_s3api_policy.sh +++ b/tests/test_s3api_policy.sh @@ -35,6 +35,7 @@ source ./tests/commands/get_object.sh source ./tests/commands/put_bucket_policy.sh source ./tests/commands/put_bucket_tagging.sh source ./tests/commands/put_object.sh +source ./tests/drivers/get_bucket_acl/get_bucket_acl_s3api.sh export RUN_USERS=true diff --git a/tests/test_s3api_policy_bucket.sh b/tests/test_s3api_policy_bucket.sh index 1d514e7..8a08bf3 100644 --- a/tests/test_s3api_policy_bucket.sh +++ b/tests/test_s3api_policy_bucket.sh @@ -15,7 +15,6 @@ # under the License. source ./tests/commands/put_public_access_block.sh -source ./tests/util/util_acl.sh test_s3api_policy_delete_bucket_policy() { policy_file="policy_file" @@ -28,6 +27,7 @@ test_s3api_policy_delete_bucket_policy() { run setup_user_v2 "user" 1 "$BUCKET_ONE_NAME" assert_success + # shellcheck disable=SC2154 user_id=${lines[0]} username=${lines[1]} password=${lines[2]} diff --git a/tests/util/util_acl.sh b/tests/util/util_acl.sh deleted file mode 100644 index 501bc06..0000000 --- a/tests/util/util_acl.sh +++ /dev/null @@ -1,408 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2024 Versity Software -# This file is licensed under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source ./tests/util/util_users.sh - -get_check_default_acl_s3cmd() { - if [ $# -ne 1 ]; then - log 2 "'get_check_acl_s3cmd' requires bucket name" - return 1 - fi - if ! get_bucket_acl "s3cmd" "$BUCKET_ONE_NAME"; then - log 2 "error retrieving acl" - return 1 - fi - # shellcheck disable=SC2154 - log 5 "Initial ACLs: $acl" - acl_line=$(echo "$acl" | grep "ACL") - user_id=$(echo "$acl_line" | awk '{print $2}') - if [[ $DIRECT == "true" ]]; then - if [[ $user_id != "$DIRECT_DISPLAY_NAME:" ]]; then - log 2 "ID mismatch ($user_id, $DIRECT_DISPLAY_NAME)" - return 1 - fi - else - if [[ $user_id != "$AWS_ACCESS_KEY_ID:" ]]; then - log 2 "ID mismatch ($user_id, $AWS_ACCESS_KEY_ID)" - return 1 - fi - fi - permission=$(echo "$acl_line" | awk '{print $3}') - if [[ $permission != "FULL_CONTROL" ]]; then - log 2 "Permission mismatch ($permission)" - return 1 - fi - return 0 -} - -get_check_post_change_acl_s3cmd() { - if [ $# -ne 1 ]; then - log 2 "'get_check_post_change_acl_s3cmd' requires bucket name" - return 1 - fi - if ! get_bucket_acl "s3cmd" "$1"; then - log 2 "error retrieving acl" - return 1 - fi - log 5 "ACL after read put: $acl" - acl_lines=$(echo "$acl" | grep "ACL") - log 5 "ACL lines: $acl_lines" - lines=() - while IFS= read -r line; do - lines+=("$line") - done <<< "$acl_lines" - log 5 "lines: ${lines[*]}" - if [[ ${#lines[@]} -ne 2 ]]; then - log 2 "unexpected number of ACL lines: ${#lines[@]}" - return 1 - fi - anon_name=$(echo "${lines[1]}" | awk '{print $2}') - anon_permission=$(echo "${lines[1]}" | awk '{print $3}') - if [[ $anon_name != "*anon*:" ]]; then - log 2 "unexpected anon name: $anon_name" - return 1 - fi - if [[ $anon_permission != "READ" ]]; then - log 2 "unexpected anon permission: $anon_permission" - return 1 - fi - return 0 -} - -get_check_acl_id() { - if [ $# -ne 2 ]; then - log 2 "'get_check_acl_id' requires client, bucket" - return 1 - fi - if ! get_bucket_acl "$1" "$2"; then - log 2 "error retrieving acl" - return 1 - fi - log 5 "Initial ACLs: $acl" - if ! id=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Owner.ID' 2>&1); then - log 2 "error getting ID: $id" - return 1 - fi - if [[ $id != "$AWS_ACCESS_KEY_ID" ]]; then - # for direct, ID is canonical user ID rather than AWS_ACCESS_KEY_ID - if ! canonical_id=$(aws --no-verify-ssl s3api list-buckets --query 'Owner.ID' 2>&1); then - log 2 "error getting canonical ID: $canonical_id" - return 1 - fi - canonical_id="$(echo -n "$canonical_id" | grep -v "InsecureRequestWarning" | sed "s/\"//g")" - log 5 "canonical ID: $canonical_id" - if [[ $id != "$canonical_id" ]]; then - log 2 "acl ID doesn't match AWS key or canonical ID" - return 1 - fi - fi - return 0 -} - -get_check_acl_after_first_put() { - if [ $# -ne 2 ]; then - log 2 "'get_check_acl_after_first_put' requires client, bucket" - return 1 - fi - if ! get_bucket_acl "$1" "$BUCKET_ONE_NAME"; then - log 2 "error retrieving second ACL" - return 1 - fi - log 5 "Acls after 1st put: $acl" - if ! public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants[1]' 2>&1); then - log 2 "error getting public grants: $public_grants" - return 1 - fi - if ! permission=$(echo "$public_grants" | jq -r '.Permission' 2>&1); then - log 2 "error getting permission: $permission" - return 1 - fi - if [[ $permission != "READ" ]]; then - log 2 "incorrect permission ($permission)" - return 1 - fi - return 0 -} - -get_check_acl_after_second_put() { - if [ $# -ne 2 ]; then - log 2 "'get_check_acl_after_second_put' requires client, bucket" - return 1 - fi - if ! get_bucket_acl "$1" "$BUCKET_ONE_NAME"; then - log 2 "error retrieving third ACL" - return 1 - fi - if ! public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants' 2>&1); then - log 2 "error retrieving public grants: $public_grants" - return 1 - fi - if ! public_grant_length=$(echo "$public_grants" | jq -r 'length' 2>&1); then - log 2 "Error retrieving public grant length: $public_grant_length" - return 1 - fi - if [[ $public_grant_length -ne 2 ]]; then - log 2 "incorrect grant length for private ACL ($public_grant_length)" - return 1 - fi - if ! permission=$(echo "$public_grants" | jq -r '.[0].Permission' 2>&1); then - log 2 "Error retrieving permission: $permission" - return 1 - fi - if [[ $permission != "FULL_CONTROL" ]]; then - log 2 "incorrect permission ($permission)" - return 1 - fi - return 0 -} - -get_check_acl_after_policy() { - if [ $# -ne 1 ]; then - log 2 "'get_check_acl_after_policy' requires bucket name" - return 1 - fi - if ! get_bucket_acl "s3api" "$1"; then - log 2 "error getting bucket acl" - return 1 - fi - # shellcheck disable=SC2154 - log 5 "ACL: $acl" - if ! second_grant=$(echo "$acl" | jq -r ".Grants[1]" 2>&1); then - log 2 "error getting second grant: $second_grant" - return 1 - fi - if ! second_grantee=$(echo "$second_grant" | jq -r ".Grantee" 2>&1); then - log 2 "error getting second grantee: $second_grantee" - return 1 - fi - if ! permission=$(echo "$second_grant" | jq -r ".Permission" 2>&1); then - log 2 "error getting permission: $permission" - return 1 - fi - log 5 "second grantee: $second_grantee" - if [[ $permission != "READ" ]]; then - log 2 "incorrect permission: $permission" - return 1 - fi - if [[ $DIRECT == "true" ]]; then - if ! uri=$(echo "$second_grantee" | jq -r ".URI" 2>&1); then - log 2 "error getting uri: $uri" - return 1 - fi - if [[ $uri != "http://acs.amazonaws.com/groups/global/AllUsers" ]]; then - log 2 "unexpected URI: $uri" - return 1 - fi - else - if ! id=$(echo "$second_grantee" | jq -r ".ID" 2>&1); then - log 2 "error getting ID: $id" - return 1 - fi - if [[ $id != "all-users" ]]; then - log 2 "unexpected ID: $id" - return 1 - fi - fi -} - -check_direct_display_name() { - if ! display_name=$(echo "$owner" | xmllint --xpath '//*[local-name()="DisplayName"]/text()' - 2>&1); then - log 2 "error getting display name: $display_name" - return 1 - fi - if [ "$display_name" != "$DIRECT_DISPLAY_NAME" ]; then - log 2 "display name mismatch (expected '$DIRECT_DISPLAY_NAME', actual '$display_name')" - return 1 - fi -} - -get_and_check_acl_rest() { - if [ $# -ne 1 ]; then - log 2 "'get_and_check_acl_rest' requires bucket name" - return 1 - fi - if ! get_bucket_acl_rest "$1" "" "check_acl_rest"; then - log 2 "error getting and checking acl" - return 1 - fi - return 0 -} - -check_acl_rest() { - if ! check_param_count_v2 "acl file" 1 $#; then - return 1 - fi - log 5 "acl: $(cat "$1")" - if ! access_control_policy=$(xmllint --xpath '//*[local-name()="AccessControlPolicy"]' "$1" 2>&1); then - log 2 "error getting access control policy: $access_control_policy" - return 1 - fi - if ! owner=$(echo "$access_control_policy" | xmllint --xpath '//*[local-name()="Owner"]' - 2>&1); then - log 2 "error getting owner information: $owner" - return 1 - fi - if [ "$DIRECT" == "true" ]; then - if ! check_direct_display_name; then - log 2 "error checking direct display name" - return 1 - fi - else - if ! id=$(echo "$owner" | xmllint --xpath '//*[local-name()="ID"]/text()' - 2>&1); then - log 2 "error getting ID: $id" - return 1 - fi - if [ "$id" != "$AWS_ACCESS_KEY_ID" ]; then - log 2 "ID mismatch" - return 1 - fi - fi - return 0 -} - -setup_acl() { - if ! check_param_count "setup_acl" "acl file, grantee type, grantee, permission, owner ID" 5 $#; then - return 1 - fi - cat < "$1" - - - $5 - - - - - $3 - - $4 - - - -EOF -} - -setup_acl_json() { - if [ $# -ne 5 ]; then - log 2 "'setup_acl_json' requires acl file, grantee type, grantee ID, permission, owner ID" - return 1 - fi - cat < "$1" -{ - "Grants": [ - { - "Grantee": { - "Type": "$2", - "ID": "$3" - }, - "Permission": "$4" - } - ], - "Owner": { - "ID": "$5" - } -} -EOF -} - -create_versitygw_acl_user_or_get_direct_user() { - if [ $# -ne 2 ]; then - log 2 "'create_versitygw_acl_user_or_get_direct_user' requires username, password" - return 1 - fi - if [ "$DIRECT" == "true" ]; then - if [ -z "$AWS_CANONICAL_ID" ] || [ -z "$ACL_AWS_CANONICAL_ID" ] || [ -z "$ACL_AWS_ACCESS_KEY_ID" ] || [ -z "$ACL_AWS_SECRET_ACCESS_KEY" ]; then - log 2 "direct ACL calls require the following env vars: AWS_CANONICAL_ID, ACL_AWS_CANONICAL_ID, ACL_AWS_ACCESS_KEY_ID, ACL_AWS_SECRET_ACCESS_KEY" - return 1 - fi - owner_canonical_id="$AWS_CANONICAL_ID" - user_canonical_id="$ACL_AWS_CANONICAL_ID" - username="$ACL_AWS_ACCESS_KEY_ID" - password="$ACL_AWS_SECRET_ACCESS_KEY" - else - owner_canonical_id="$AWS_ACCESS_KEY_ID" - if ! create_user_versitygw "$1" "$2" "user"; then - log 2 "error creating versitygw user" - return 1 - fi - # shellcheck disable=SC2154 - user_canonical_id="$1" - username="$1" - # shellcheck disable=SC2154 - password="$2" - fi - echo "$owner_canonical_id" - echo "$user_canonical_id" - echo "$username" - echo "$password" -} - -put_invalid_acl_rest_verify_failure() { - if [ $# -ne 2 ]; then - log 2 "'put_invalid_acl_rest_verify_failure' requires bucket name, ACL file" - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" ACL_FILE="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_bucket_acl.sh); then - log 2 "error attempting to put bucket acl: $result" - return 1 - fi - if [ "$result" != "400" ]; then - log 2 "response returned code: $result (error: $(cat "$TEST_FILE_FOLDER/response.txt"))" - return 1 - fi - if ! error_code=$(xmllint --xpath '//*[local-name()="Code"]/text()' "$TEST_FILE_FOLDER/response.txt" 2>&1); then - log 2 "error getting display name: $error_code" - return 1 - fi - if [ "$error_code" != "MalformedACLError" ]; then - log 2 "invalid error code, expected 'MalformedACLError', was '$error_code'" - return 1 - fi - return 0 -} - -# param: bucket name -# return 0 for success, 1 for failure -check_ownership_rule_and_reset_acl() { - if [ $# -ne 1 ]; then - log 2 "'check_ownership_rule_and_reset_acl' requires bucket name" - return 1 - fi - if ! object_ownership_rule=$(get_bucket_ownership_controls_rest "$1" 2>&1); then - log 2 "error getting bucket ownership controls" - return 1 - fi - log 5 "ownership rule: $object_ownership_rule" - if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then - log 2 "error resetting bucket ACL" - return 1 - fi -} - -# get object acl -# param: object path -# export acl for success, return 1 for error -get_object_acl() { - if [ $# -ne 2 ]; then - log 2 "object ACL command missing object name" - return 1 - fi - local exit_code=0 - acl=$(aws --no-verify-ssl s3api get-object-acl --bucket "$1" --key "$2" 2>&1) || exit_code="$?" - if [ $exit_code -ne 0 ]; then - log 2 "Error getting object ACLs: $acl" - return 1 - fi - export acl -} diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh index f463be9..3ec9acc 100644 --- a/tests/util/util_bucket.sh +++ b/tests/util/util_bucket.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash +source ./tests/drivers/delete_object/delete_object_rest.sh source ./tests/drivers/params.sh source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh source ./tests/drivers/list_objects/list_objects_rest.sh -source ./tests/util/util_acl.sh +source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh source ./tests/util/util_multipart_abort.sh source ./tests/util/util_policy.sh source ./tests/util/util_retention.sh diff --git a/tests/util/util_file.sh b/tests/util/util_file.sh index 5cf1233..72f95cb 100644 --- a/tests/util/util_file.sh +++ b/tests/util/util_file.sh @@ -209,12 +209,21 @@ compare_files() { # fail on error create_large_file() { log 6 "create_large_file" - if [ $# -ne 1 ]; then - log 2 "'create_large_file' requires file name" + if ! check_param_count_v2 "file name" 1 $#; then return 1 fi + if ! create_large_file_with_size "$1" 160; then + log 2 "error creating 160MB file" + return 1 + fi + return 0 +} - filesize=$((160*1024*1024)) +create_large_file_with_size() { + if ! check_param_count_v2 "file name, size in MB" 2 $#; then + return 1 + fi + filesize=$(($2*1024*1024)) if ! error=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER"/"$1" bs=1024 count=$((filesize/1024)) 2>&1); then log 2 "error adding data to large file: $error" return 1 @@ -222,6 +231,20 @@ create_large_file() { return 0 } +create_and_split_large_file() { + if ! check_param_count_v2 "file name, size in MB, pieces" 3 $#; then + return 1 + fi + if ! create_large_file_with_size "$1" "$2"; then + log 2 "error creating large file" + return 1 + fi + if ! split_file "$TEST_FILE_FOLDER/$1" "$3"; then + log 2 "error splitting file" + return 1 + fi +} + # param: number of files # fail on error create_test_file_count() { diff --git a/tests/util/util_list_objects.sh b/tests/util/util_list_objects.sh index ea787c0..344925e 100644 --- a/tests/util/util_list_objects.sh +++ b/tests/util/util_list_objects.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash source ./tests/commands/list_objects_v2.sh -source ./tests/util/util_xml.sh +source ./tests/drivers/xml.sh # Copyright 2024 Versity Software # This file is licensed under the Apache License, Version 2.0 diff --git a/tests/util/util_versioning.sh b/tests/util/util_versioning.sh index fa6f6d7..909f815 100644 --- a/tests/util/util_versioning.sh +++ b/tests/util/util_versioning.sh @@ -23,59 +23,6 @@ check_if_versioning_enabled() { return 1 } -delete_old_versions() { - if ! check_param_count "delete_old_versions" "bucket" 1 $#; then - return 1 - fi - if ! list_object_versions "rest" "$1"; then - log 2 "error listing object versions" - return 1 - fi - # shellcheck disable=SC2154 - log 5 "versions: $versions" - version_keys=() - version_ids=() - - if ! parse_version_data_by_type "rest" "$2"; then - log 2 "error parsing version data" - return 1 - fi - - log 5 "version keys: ${version_keys[*]}" - log 5 "version IDs: ${version_ids[*]}" - for idx in "${!version_keys[@]}"; do - if ! delete_object_version_with_or_without_retention "$1"; then - log 2 "error deleting version with or without retention" - return 1 - fi - done -} - -delete_object_version_with_or_without_retention() { - if ! check_param_count "delete_object_version_with_or_without_retention" "bucket" 1 $#; then - return 1 - fi - log 5 "idx: $idx" - log 5 "version ID: ${version_ids[$idx]}" - # shellcheck disable=SC2154 - if [ "$lock_config_exists" == "true" ]; then - if ! check_remove_legal_hold_versions "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then - log 2 "error checking, removing legal hold versions" - fi - if ! delete_object_version_rest_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then - log 2 "error deleting object version, bypassing retention" - return 1 - fi - else - if ! delete_object_version_rest "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then - log 2 "error deleting object version" - return 1 - fi - fi - log 5 "successfully deleted version with key '${version_keys[$idx]}', id '${version_ids[$idx]}'" - return 0 -} - parse_version_data_by_type() { if ! check_param_count "parse_version_data_by_type" "client, data" 2 $#; then return 1