diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 59ab268..bb7659a 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -15,120 +15,140 @@ jobs: RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "mc, posix, file count, non-static, folder IAM" IAM_TYPE: folder RUN_SET: "mc-file-count" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "REST, posix, non-static, base|acl|multipart|put-object, folder IAM" IAM_TYPE: folder RUN_SET: "rest-base,rest-acl,rest-multipart,rest-put-object" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "REST, posix, non-static, chunked|checksum|versioning|bucket, folder IAM" IAM_TYPE: folder RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket,rest-list-buckets,rest-create-bucket,rest-head-bucket" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "REST, posix, non-static, not implemented|rest-delete-bucket-ownership-controls|rest-delete-bucket-tagging, folder IAM" IAM_TYPE: folder RUN_SET: "rest-not-implemented,rest-delete-bucket-ownership-controls,rest-delete-bucket-tagging" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "REST, posix, static, base|acl|multipart|put-object, folder IAM" IAM_TYPE: folder RUN_SET: "rest-base,rest-acl,rest-multipart,rest-put-object" RECREATE_BUCKETS: "false" DELETE_BUCKETS_AFTER_TEST: "false" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "REST, posix, static, chunked|checksum|versioning|bucket, folder IAM" IAM_TYPE: folder RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket,rest-list-buckets,rest-create-bucket,rest-head-bucket" RECREATE_BUCKETS: "false" DELETE_BUCKETS_AFTER_TEST: "false" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "REST, posix, static, not implemented|rest-delete-bucket-ownership-controls|rest-delete-bucket-tagging, folder IAM" IAM_TYPE: folder RUN_SET: "rest-not-implemented,rest-delete-bucket-ownership-controls,rest-delete-bucket-tagging" RECREATE_BUCKETS: "false" DELETE_BUCKETS_AFTER_TEST: "false" BACKEND: "posix" - - set: "REST, posix, static, rest-put-bucket-tagging, folder IAM" + AWS_REGION: "us-east-1" + - set: "REST, posix, static, rest-put-bucket-tagging|rest-get-bucket-location, folder IAM" IAM_TYPE: folder - RUN_SET: "rest-put-bucket-tagging" + RUN_SET: "rest-put-bucket-tagging,rest-get-bucket-location" RECREATE_BUCKETS: "false" DELETE_BUCKETS_AFTER_TEST: "false" BACKEND: "posix" - - set: "REST, posix, non-static, rest-put-bucket-tagging, folder IAM" + AWS_REGION: "us-east-1" + - set: "REST, posix, non-static, rest-put-bucket-tagging|rest-get-bucket-location, folder IAM" IAM_TYPE: folder - RUN_SET: "rest-put-bucket-tagging" + RUN_SET: "rest-put-bucket-tagging,rest-get-bucket-location" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-west-1" - set: "s3, posix, non-file count, non-static, folder IAM" IAM_TYPE: folder RUN_SET: "s3-non-file-count" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3, posix, file count, non-static, folder IAM" IAM_TYPE: folder RUN_SET: "s3-file-count" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM" IAM_TYPE: folder RUN_SET: "s3api-bucket,s3api-object,s3api-multipart" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3api, posix, policy, non-static, folder IAM" IAM_TYPE: folder RUN_SET: "s3api-policy" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3api, posix, user, non-static, s3 IAM" IAM_TYPE: s3 RUN_SET: "s3api-user" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3api, posix, bucket, static, folder IAM" IAM_TYPE: folder RUN_SET: "s3api-bucket" RECREATE_BUCKETS: "false" DELETE_BUCKETS_AFTER_TEST: "false" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3api, posix, multipart, static, folder IAM" IAM_TYPE: folder RUN_SET: "s3api-multipart" RECREATE_BUCKETS: "false" DELETE_BUCKETS_AFTER_TEST: "false" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3api, posix, object, static, folder IAM" IAM_TYPE: folder RUN_SET: "s3api-object" RECREATE_BUCKETS: "false" DELETE_BUCKETS_AFTER_TEST: "false" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3api, posix, policy, static, folder IAM" IAM_TYPE: folder RUN_SET: "s3api-policy" RECREATE_BUCKETS: "false" DELETE_BUCKETS_AFTER_TEST: "false" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3api, posix, user, static, folder IAM" IAM_TYPE: folder RUN_SET: "s3api-user" RECREATE_BUCKETS: "false" DELETE_BUCKETS_AFTER_TEST: "false" BACKEND: "posix" + AWS_REGION: "us-east-1" # TODO fix/debug s3 gateway #- set: "s3api, s3, multipart|object, non-static, folder IAM" # IAM_TYPE: folder @@ -146,24 +166,28 @@ jobs: RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3cmd, posix, non-user, non-static, folder IAM" IAM_TYPE: folder RUN_SET: "s3cmd-non-user" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "s3cmd, posix, user, non-static, folder IAM" IAM_TYPE: folder RUN_SET: "s3cmd-user" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" - set: "setup/remove static buckets scripts" IAM_TYPE: folder RUN_SET: "setup-remove-static" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" + AWS_REGION: "us-east-1" steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -243,11 +267,12 @@ jobs: PYTHON_ENV_FOLDER: ${{ github.workspace }}/env AUTOGENERATE_USERS: true USER_AUTOGENERATION_PREFIX: github-actions-test- + AWS_REGION: ${{ matrix.AWS_REGION }} run: | make testbin export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn - export AWS_REGION=us-east-1 + export AWS_REGION=$AWS_REGION export AWS_ACCESS_KEY_ID_TWO=user export AWS_SECRET_ACCESS_KEY_TWO=pass export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED diff --git a/tests/commands/get_bucket_location.sh b/tests/commands/get_bucket_location.sh index c106284..5b77c12 100644 --- a/tests/commands/get_bucket_location.sh +++ b/tests/commands/get_bucket_location.sh @@ -81,4 +81,19 @@ get_bucket_location_mc() { # shellcheck disable=SC2034 bucket_location=$(echo "$info" | grep -o 'Location:.*' | awk '{print $2}') return 0 +} + +get_bucket_location_rest() { + if ! check_param_count_v2 "bucket, callback" 2 $#; then + return 1 + fi + if ! send_rest_go_command "200" "-bucketName" "$1" "-method" "GET" "-query" "location=" "-awsRegion" "$AWS_REGION"; then + log 2 "error sending rest go command" + return 1 + fi + if [ "$2" != "" ] && ! "$2" "$TEST_FILE_FOLDER/result.txt"; then + log 2 "callback error" + return 1 + fi + return 0 } \ No newline at end of file diff --git a/tests/commands/get_bucket_policy.sh b/tests/commands/get_bucket_policy.sh index 4284479..b9c9425 100644 --- a/tests/commands/get_bucket_policy.sh +++ b/tests/commands/get_bucket_policy.sh @@ -109,6 +109,7 @@ get_bucket_policy_rest() { if ! check_param_count "get_bucket_policy_rest" "bucket" 1 $#; then return 1 fi + log 5 "aws region: $AWS_REGION" if ! get_bucket_policy_rest_expect_code "$1" "200"; then log 2 "error getting REST bucket policy" return 1 diff --git a/tests/util/util_aws.sh b/tests/drivers/abort_multipart_upload/abort_multipart_upload_s3api.sh similarity index 83% rename from tests/util/util_aws.sh rename to tests/drivers/abort_multipart_upload/abort_multipart_upload_s3api.sh index 212d6f0..4b1b3f1 100644 --- a/tests/util/util_aws.sh +++ b/tests/drivers/abort_multipart_upload/abort_multipart_upload_s3api.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright 2024 Versity Software +# Copyright 2025 Versity Software # This file is licensed under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at @@ -52,17 +52,3 @@ abort_all_multipart_uploads() { done <<< "$lines" return 0 } - -remove_insecure_request_warning() { - if [[ $# -ne 1 ]]; then - log 2 "remove insecure request warning requires input lines" - return 1 - fi - parsed_output=() - while IFS= read -r line; do - if [[ $line != *InsecureRequestWarning* ]]; then - parsed_output+=("$line") - fi - done <<< "$1" - export parsed_output -} diff --git a/tests/drivers/create_bucket/create_bucket_rest.sh b/tests/drivers/create_bucket/create_bucket_rest.sh index cce553f..78d7bd4 100644 --- a/tests/drivers/create_bucket/create_bucket_rest.sh +++ b/tests/drivers/create_bucket/create_bucket_rest.sh @@ -17,6 +17,7 @@ source ./tests/drivers/delete_bucket/delete_bucket_rest.sh source ./tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh source ./tests/drivers/get_object/get_object_rest.sh +source ./tests/drivers/list_objects/list_objects_rest.sh source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh source ./tests/drivers/put_object/put_object_rest.sh source ./tests/drivers/user.sh @@ -235,3 +236,76 @@ setup_bucket_object_lock_enabled() { fi return 0 } + +send_curl_command_create_bucket_expect_error() { + if ! check_param_count_gt "response code, error code, message, params" 4 $#; then + return 1 + fi + if ! send_curl_command_create_bucket_expect_error_callback "$1" "$2" "$3" "" "${@:4}"; then + log 2 "error sending curl create bucket command" + return 1 + fi + return 0 +} + +send_curl_command_create_bucket_expect_error_callback() { + if ! check_param_count_gt "response code, error code, message, callback, params" 5 $#; then + return 1 + fi + if ! bucket_name=$(get_bucket_name "$BUCKET_ONE_NAME" 2>&1); then + log 2 "error getting bucket name from '$BUCKET_ONE_NAME': $bucket_name" + return 1 + fi + if ! send_rest_go_command_expect_error "$1" "$2" "$3" "-bucketName" "$bucket_name" "-commandType" "createBucket" "${@:5}"; then + log 2 "error sending rest go command" + return 1 + fi + if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/result.txt"; then + log 2 "callback error" + return 1 + fi + return 0 +} + +send_invalid_location_constraint_check_error() { + if ! check_param_count_v2 "invalid param" 1 $#; then + return 1 + fi + invalid_location_constraint="$1" + if ! send_curl_command_create_bucket_expect_error_callback "400" "InvalidLocationConstraint" "The specified location-constraint is not valid" \ + "check_location_constraint_param" "-locationConstraint" "$invalid_location_constraint"; then + log 2 "error sending curl command and checking callback" + return 1 + fi + return 0 +} + +check_location_constraint_param() { + if ! check_param_count_v2 "file" 1 $#; then + return 1 + fi + if ! check_error_parameter "$1" "LocationConstraint" "$invalid_location_constraint"; then + log 2 "location constraint mismatch" + return 1 + fi + return 0 +} + +create_bucket_and_run_command() { + if ! check_param_count_gt "bucket, command, params" 2 $#; then + return 1 + fi + if ! bucket_name=$(get_bucket_name "$1" 2>&1); then + log 2 "error creating bucket '$1': $bucket_name" + return 1 + fi + if ! setup_bucket "$bucket_name"; then + log 2 "error setting up bucket" + return 1 + fi + if ! "$2" "$bucket_name" "${@:3}"; then + log 2 "error running command on bucket" + return 1 + fi + return 0 +} diff --git a/tests/drivers/delete_bucket/delete_bucket_rest.sh b/tests/drivers/delete_bucket/delete_bucket_rest.sh index d16146c..25cb89f 100644 --- a/tests/drivers/delete_bucket/delete_bucket_rest.sh +++ b/tests/drivers/delete_bucket/delete_bucket_rest.sh @@ -14,8 +14,12 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/get_bucket_location.sh source ./tests/commands/list_buckets.sh +source ./tests/drivers/get_bucket_location/get_bucket_location_rest.sh +source ./tests/drivers/head_bucket/head_bucket_rest.sh source ./tests/drivers/list_buckets/list_buckets_rest.sh +source ./tests/drivers/user.sh delete_buckets_with_prefix() { if ! check_param_count_v2 "bucket prefix" 1 $#; then @@ -129,6 +133,12 @@ delete_bucket_recursive() { return 1 fi + if ! location=$(get_bucket_location_rest "$1" "parse_bucket_location" 2>&1); then + log 2 "error getting bucket location: $location" + return 1 + fi + log 5 "location: $location" + if ! reset_bucket "$1"; then log 2 "error clearing bucket (s3api)" return 1 diff --git a/tests/drivers/file.sh b/tests/drivers/file.sh index 0306b00..3c8f316 100644 --- a/tests/drivers/file.sh +++ b/tests/drivers/file.sh @@ -15,6 +15,7 @@ # under the License. source ./tests/drivers/params.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh setup_bucket_and_file() { if ! check_param_count_v2 "bucket, file name" 2 $#; then diff --git a/tests/drivers/get_bucket_location/get_bucket_location_rest.sh b/tests/drivers/get_bucket_location/get_bucket_location_rest.sh new file mode 100644 index 0000000..52d13af --- /dev/null +++ b/tests/drivers/get_bucket_location/get_bucket_location_rest.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/drivers/xml.sh + +get_check_bucket_location() { + if ! check_param_count_v2 "bucket name, expected value" 2 $#; then + return 1 + fi + expected_location="$2" + if [ "$expected_location" == "us-east-1" ]; then + expected_location="" + fi + if ! get_bucket_location_rest "$1" "check_location_constraint"; then + log 2 "error getting and checking bucket location" + return 1 + fi + return 0 +} + +check_location_constraint() { + if ! check_param_count_v2 "file" 1 $#; then + return 1 + fi + log 5 "location constraint: $(cat "$1")" + if ! location_constraint=$(get_element_text "$1" "LocationConstraint" 2>&1); then + log 2 "error getting location constraint: $location_constraint" + return 1 + fi + if [ "$location_constraint" != "$expected_location" ]; then + log 2 "expected location constraint of '$expected_location', was '$location_constraint'" + return 1 + fi + return 0 +} + +parse_bucket_location() { + if ! check_param_count_v2 "file" 1 $#; then + return 1 + fi + log 5 "data: $(cat "$1")" + if ! location_constraint=$(get_element_text "$1" "LocationConstraint" 2>&1); then + log 2 "error getting location constraint: $location_constraint" + return 1 + fi + echo "$location_constraint" + return 0 +} diff --git a/tests/drivers/head_bucket/head_bucket_rest.sh b/tests/drivers/head_bucket/head_bucket_rest.sh new file mode 100644 index 0000000..932e117 --- /dev/null +++ b/tests/drivers/head_bucket/head_bucket_rest.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Copyright 2025 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +bucket_exists() { + if ! check_param_count "bucket_exists" "bucket name" 1 $#; then + return 2 + fi + local exists=0 + head_bucket "rest" "$1" || exists=$? + log 5 "bucket exists response code: $exists" + # shellcheck disable=SC2181 + if [ $exists -eq 2 ]; then + log 2 "unexpected error checking if bucket exists" + return 2 + fi + if [ $exists -eq 0 ]; then + return 0 + fi + return 1 +} diff --git a/tests/drivers/head_bucket/head_bucket_s3api.sh b/tests/drivers/head_bucket/head_bucket_s3api.sh new file mode 100644 index 0000000..a91085c --- /dev/null +++ b/tests/drivers/head_bucket/head_bucket_s3api.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Copyright 2025 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +bucket_is_accessible() { + if ! check_param_count "bucket_is_accessible" "bucket" 1 $#; then + return 2 + fi + local exit_code=0 + local error + error=$(aws --no-verify-ssl s3api head-bucket --bucket "$1" 2>&1) || exit_code="$?" + if [ $exit_code -eq 0 ]; then + return 0 + fi + if [[ "$error" == *"500"* ]]; then + return 1 + fi + log 2 "Error checking bucket accessibility: $error" + return 2 +} + +check_for_empty_region() { + if ! check_param_count "check_for_empty_region" "bucket" 1 $#; then + return 2 + fi + if ! head_bucket "s3api" "$BUCKET_ONE_NAME"; then + log 2 "error getting bucket info" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "INFO: $bucket_info" + if ! region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1); then + log 2 "error getting region: $region" + return 1 + fi + if [[ $region == "" ]]; then + log 2 "empty bucket region" + return 1 + fi + return 0 +} diff --git a/tests/drivers/not_implemented/not_implemented_rest.sh b/tests/drivers/not_implemented/not_implemented_rest.sh index c620b80..1444d18 100644 --- a/tests/drivers/not_implemented/not_implemented_rest.sh +++ b/tests/drivers/not_implemented/not_implemented_rest.sh @@ -15,6 +15,7 @@ # under the License. source ./tests/rest_scripts/rest.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh send_not_implemented_expect_failure() { if [ $(($# % 2)) -ne 0 ]; then diff --git a/tests/drivers/openssl.sh b/tests/drivers/openssl.sh index 4b6cf3c..e60cfb2 100644 --- a/tests/drivers/openssl.sh +++ b/tests/drivers/openssl.sh @@ -133,7 +133,7 @@ send_openssl_go_command() { if ! check_param_count_gt "expected HTTP code, params" 2 $#; then return 1 fi - if ! go run "./tests/rest_scripts/generateCommand.go" "-awsAccessKeyId" "$AWS_ACCESS_KEY_ID" "-awsSecretAccessKey" "$AWS_SECRET_ACCESS_KEY" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/openssl_command.txt" "${@:2}"; then + if ! go run "./tests/rest_scripts/generateCommand.go" "-awsAccessKeyId" "$AWS_ACCESS_KEY_ID" "-awsSecretAccessKey" "$AWS_SECRET_ACCESS_KEY" "-awsRegion" "$AWS_REGION" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/openssl_command.txt" "${@:2}"; then log 2 "error sending go command and checking error" return 1 fi diff --git a/tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh b/tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh new file mode 100644 index 0000000..a6d4099 --- /dev/null +++ b/tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Copyright 2025 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/drivers/head_bucket/head_bucket_rest.sh + +# params: client, bucket name(s) +# return 0 for success, 1 for failure +setup_buckets() { + if ! check_param_count_gt "minimum of 1 bucket name" 1 $#; then + return 1 + fi + for name in "$@"; do + if ! setup_bucket "$name"; then + log 2 "error setting up bucket $name" + return 1 + fi + done + return 0 +} + +# params: client, bucket name +# return 0 on successful setup, 1 on error +setup_bucket() { + log 6 "setup_bucket" + if ! check_param_count "setup_bucket" "bucket name" 1 $#; then + return 1 + fi + + bucket_exists="true" + if ! bucket_exists "$1"; then + if [[ $RECREATE_BUCKETS == "false" ]]; then + log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user" + return 1 + fi + bucket_exists="false" + fi + + if ! bucket_cleanup_if_bucket_exists "$1" "$bucket_exists"; then + log 2 "error deleting bucket or contents if they exist" + return 1 + fi + + log 5 "util.setup_bucket: bucket name: $1" + if [[ $RECREATE_BUCKETS == "true" ]]; then + if ! create_bucket_rest_expect_success "$1" ""; then + log 2 "error creating bucket" + return 1 + fi + else + log 5 "skipping bucket re-creation" + fi + + if [[ $1 == "s3cmd" ]]; then + log 5 "putting bucket ownership controls" + if bucket_exists "$1" && ! put_bucket_ownership_controls "$1" "BucketOwnerPreferred"; then + log 2 "error putting bucket ownership controls" + return 1 + fi + fi + return 0 +} diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 7e3e14f..d3ae9a7 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -171,7 +171,7 @@ send_rest_command_expect_success_callback() { } rest_go_command_perform_send() { - if ! curl_command=$(go run ./tests/rest_scripts/generateCommand.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -url "$AWS_ENDPOINT_URL" "$@" 2>&1); then + if ! curl_command=$(go run ./tests/rest_scripts/generateCommand.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -awsRegion "$AWS_REGION" -url "$AWS_ENDPOINT_URL" "$@" 2>&1); then log 2 "error: $curl_command" return 1 fi diff --git a/tests/drivers/user.sh b/tests/drivers/user.sh index c99ffc4..487178a 100644 --- a/tests/drivers/user.sh +++ b/tests/drivers/user.sh @@ -14,7 +14,13 @@ # specific language governing permissions and limitations # under the License. +source ./tests/drivers/delete_object/delete_object_rest.sh +source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh +source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh source ./tests/drivers/file.sh +source ./tests/util/util_multipart_abort.sh +source ./tests/util/util_policy.sh +source ./tests/util/util_retention.sh source ./tests/util/util_users.sh create_versitygw_acl_user_or_get_direct_user() { @@ -112,3 +118,55 @@ setup_bucket_file_and_user_v2() { echo "$result" return 0 } + +reset_bucket() { + if ! check_param_count "reset_bucket" "bucket" 1 $#; then + return 1 + fi + log 6 "reset bucket '$1'" + + if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$1"; then + log 3 "error logging bucket policy" + fi + + if ! check_object_lock_config "$1"; then + log 2 "error checking object lock config" + return 1 + fi + + if [[ "$DIRECT" != "true" ]] && ! add_governance_bypass_policy "$1"; then + log 2 "error adding governance bypass policy" + return 1 + fi + + if ! list_and_delete_objects "$1"; then + log 2 "error listing and deleting objects" + return 1 + fi + + if ! abort_all_multipart_uploads_rest "$1"; then + log 2 "error aborting all multipart uploads" + return 1 + fi + + if [ "$SKIP_ACL_TESTING" != "true" ] && ! check_ownership_rule_and_reset_acl "$1"; then + log 2 "error checking ownership rule and resetting acl" + return 1 + fi + + if ! delete_bucket_policy_rest "$1"; then + log 2 "error deleting bucket policy" + return 1 + fi + + # shellcheck disable=SC2154 + if [[ $lock_config_exists == true ]] && ! remove_retention_policy_rest "$1"; then + log 2 "error removing bucket retention policy" + return 1 + fi + + if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID"; then + log 2 "error changing bucket owner back to root" + return 1 + fi +} diff --git a/tests/drivers/xml.sh b/tests/drivers/xml.sh index cc5c38a..ab33dad 100644 --- a/tests/drivers/xml.sh +++ b/tests/drivers/xml.sh @@ -68,15 +68,17 @@ get_element_text() { log 5 "data: $(cat "$1")" log 5 "xpath: $xpath" - xml_data="$(grep '<[^/][^ >]*>' "$1")" - log 5 "XML data: $xml_data" - log 5 "result: $(echo -n "$xml_data" | xmllint --xpath "boolean($xpath)" - 2>&1)" - result=$(echo -n "$xml_data" | xmllint --xpath "boolean($xpath)" - 2>&1) + if ! get_xml_data "$1" "$1.xml"; then + log 2 "error getting XML data" + return 1 + fi + log 5 "result: $(xmllint --xpath "boolean($xpath)" "$1.xml" 2>&1)" + result=$(xmllint --xpath "boolean($xpath)" "$1.xml" 2>&1) if [ "$result" == "false" ]; then log 2 "element matching '$xpath' doesn't exist" return 1 fi - if ! xml_val=$(echo -n "$xml_data" | xmllint --xpath "${xpath}/text()" - 2>/dev/null); then + if ! xml_val=$(xmllint --xpath "${xpath}/text()" "$1.xml" 2>/dev/null); then echo "" return 0 fi diff --git a/tests/rest_scripts/command/createBucketCommand.go b/tests/rest_scripts/command/createBucketCommand.go new file mode 100644 index 0000000..f6eca8b --- /dev/null +++ b/tests/rest_scripts/command/createBucketCommand.go @@ -0,0 +1,45 @@ +package command + +import ( + "encoding/xml" + "errors" + "fmt" +) + +type CreateBucketCommandXML struct { + XMLName xml.Name `xml:"CreateBucketConfiguration"` + XMLNamespace string `xml:"xmlns,attr"` + LocationConstraint string `xml:"LocationConstraint"` +} + +type CreateBucketCommand struct { + *S3Command + Config *CreateBucketCommandXML +} + +func NewCreateBucketCommand(s3Command *S3Command, locationConstraint string, constraintSet bool) (*CreateBucketCommand, error) { + if s3Command.BucketName == "" { + return nil, errors.New("CreateBucket must have bucket name") + } + s3Command.Method = "PUT" + s3Command.Query = "" + var config *CreateBucketCommandXML = nil + if constraintSet { + config = &CreateBucketCommandXML{ + XMLNamespace: "http://s3.amazonaws.com/doc/2006-03-01/", + LocationConstraint: locationConstraint, + } + } + command := &CreateBucketCommand{ + S3Command: s3Command, + Config: config, + } + if constraintSet { + xmlData, err := xml.Marshal(command.Config) + if err != nil { + return nil, fmt.Errorf("error marshalling XML: %w", err) + } + command.Payload = "\n" + string(xmlData) + } + return command, nil +} diff --git a/tests/rest_scripts/command/payload.go b/tests/rest_scripts/command/payload.go index 938fefd..8f04377 100644 --- a/tests/rest_scripts/command/payload.go +++ b/tests/rest_scripts/command/payload.go @@ -38,11 +38,15 @@ func GetBase64ChecksumLength(checksumType string) (int64, error) { func (p *Payload) GetDataSize() (int64, error) { if !p.dataSizeCalculated { - dataSize, err := p.dataSource.SourceDataByteSize() - if err != nil { - return 0, fmt.Errorf("error getting payload data size: %w", err) + if p.dataSource != nil { + dataSize, err := p.dataSource.SourceDataByteSize() + if err != nil { + return 0, fmt.Errorf("error getting payload data size: %w", err) + } + p.dataSize = dataSize + } else { + p.dataSize = 0 } - p.dataSize = dataSize p.dataSizeCalculated = true } return p.dataSize, nil diff --git a/tests/rest_scripts/command/s3Command.go b/tests/rest_scripts/command/s3Command.go index 099afc3..d5217d6 100644 --- a/tests/rest_scripts/command/s3Command.go +++ b/tests/rest_scripts/command/s3Command.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" - "errors" "fmt" logger "github.com/versity/versitygw/tests/rest_scripts/logger" "os" @@ -112,9 +111,6 @@ type S3Command struct { } func (s *S3Command) OpenSSLCommand() error { - if s.FilePath == "" { - return errors.New("for openssl command, filePath must be set") - } if err := s.prepareForBuild(); err != nil { return fmt.Errorf("error preparing for command building: %w", err) } @@ -162,9 +158,6 @@ func (s *S3Command) prepareForBuild() error { } func (s *S3Command) preparePayload() error { - if s.PayloadFile != "" && s.Payload != "" { - return fmt.Errorf("cannot have both payload and payloadFile parameters set") - } if s.PayloadFile != "" { s.dataSource = NewFileDataSource(s.PayloadFile) } else if s.Payload != "" { @@ -198,6 +191,8 @@ func (s *S3Command) initializeOpenSSLPayloadAndGetContentLength() error { streamingUnsignedPayloadTrailerImpl := NewStreamingUnsignedPayloadWithTrailer(s.dataSource, int64(s.ChunkSize), s.ChecksumType) streamingUnsignedPayloadTrailerImpl.OmitTrailerOrKey(s.OmitPayloadTrailer, s.OmitPayloadTrailerKey) s.payloadOpenSSL = streamingUnsignedPayloadTrailerImpl + case UnsignedPayload, "": + s.payloadOpenSSL = NewWholePayload(s.dataSource) default: return fmt.Errorf("unsupported OpenSSL payload type: '%s'", s.PayloadType) } @@ -407,6 +402,9 @@ func (s *S3Command) buildOpenSSLCommand() error { if _, err = file.Write(openSSLCommandBytes); err != nil { return fmt.Errorf("error writing to file: %w", err) } + if _, err := file.Write([]byte{'\r', '\n', '\r', '\n'}); err != nil { + return fmt.Errorf("error writing to file: %w", err) + } if s.PayloadFile != "" || s.Payload != "" { if err = s.writeOpenSSLPayload(file); err != nil { return fmt.Errorf("error writing openssl payload: %w", err) @@ -416,9 +414,6 @@ func (s *S3Command) buildOpenSSLCommand() error { } func (s *S3Command) writeOpenSSLPayload(file *os.File) error { - if _, err := file.Write([]byte{'\r', '\n', '\r', '\n'}); err != nil { - return fmt.Errorf("error writing to file: %w", err) - } if awsPayload, ok := s.payloadOpenSSL.(*PayloadStreamingAWS4HMACSHA256); ok { awsPayload.AddInitialSignatureAndSigningKey(s.signature, s.signingKey) } diff --git a/tests/rest_scripts/command/wholePayload.go b/tests/rest_scripts/command/wholePayload.go index f17871f..1657b5e 100644 --- a/tests/rest_scripts/command/wholePayload.go +++ b/tests/rest_scripts/command/wholePayload.go @@ -30,6 +30,9 @@ func (w *WholePayload) GetContentLength() (int64, error) { } func (w *WholePayload) WritePayload(filePath string) error { + if w.dataSource == nil { + return nil + } sourceFile, err := w.dataSource.GetReader() if err != nil { return fmt.Errorf("error creating tee reader: %w", err) diff --git a/tests/rest_scripts/config/config.go b/tests/rest_scripts/config/config.go new file mode 100644 index 0000000..d912156 --- /dev/null +++ b/tests/rest_scripts/config/config.go @@ -0,0 +1 @@ +package config diff --git a/tests/rest_scripts/generateCommand.go b/tests/rest_scripts/generateCommand.go index 79b8774..49f0a82 100644 --- a/tests/rest_scripts/generateCommand.go +++ b/tests/rest_scripts/generateCommand.go @@ -11,6 +11,7 @@ import ( ) const ( + CreateBucket = "createBucket" PutBucketTagging = "putBucketTagging" PutObject = "putObject" ) @@ -57,6 +58,9 @@ var omitPayloadTrailer *bool var omitPayloadTrailerKey *bool var omitContentLength *bool +var locationConstraint *string +var locationConstraintSet bool = false + type restParams map[string]string func (r *restParams) String() string { @@ -90,6 +94,10 @@ func main() { log.Fatalf("Error checking flags: %v", err) } + if err := validateConfig(); err != nil { + log.Fatalf("Error validating config: %v", err) + } + baseCommand := &command.S3Command{ Method: *method, Url: *url, @@ -137,6 +145,10 @@ func getS3CommandType(baseCommand *command.S3Command) (command.S3CommandConverte var s3Command command.S3CommandConverter var err error switch *commandType { + case CreateBucket: + if s3Command, err = command.NewCreateBucketCommand(baseCommand, *locationConstraint, locationConstraintSet); err != nil { + return nil, fmt.Errorf("error setting up CreateBucket command: %v", err) + } case PutBucketTagging: fields := command.PutBucketTaggingFields{ TagCount: *tagCount, @@ -148,7 +160,7 @@ func getS3CommandType(baseCommand *command.S3Command) (command.S3CommandConverte } case PutObject: if s3Command, err = command.NewPutObjectCommand(baseCommand); err != nil { - return nil, fmt.Errorf("error setting up PutBucketTagging command: %v", err) + return nil, fmt.Errorf("error setting up PutObject command: %v", err) } default: s3Command = baseCommand @@ -211,6 +223,7 @@ func checkFlags() error { omitContentLength = flag.Bool("omitContentLength", false, "Omit content length parameter") flag.Var(&tagKeys, "tagKey", "Tag key (can add multiple)") flag.Var(&tagValues, "tagValue", "Tag value (can add multiple)") + locationConstraint = flag.String("locationConstraint", "", "Location constraint for bucket creation") // Parse the flags flag.Parse() @@ -218,13 +231,35 @@ func checkFlags() error { if f.Name == "customHostParam" { customHostParamSet = true } + if f.Name == "locationConstraint" { + locationConstraintSet = true + } }) - if flag.Lookup("awsAccessKeyId").Value.String() == "" { + return nil +} + +func validateConfig() error { + if *awsAccessKeyId == "" { return fmt.Errorf("the 'awsAccessKeyId' value must be set") } - if flag.Lookup("awsSecretAccessKey").Value.String() == "" { + if *awsSecretAccessKey == "" { return fmt.Errorf("the 'awsSecretAccessKey' value must be set") } + if *payloadFile != "" && *payload != "" { + return fmt.Errorf("cannot have both payload and payloadFile parameters set") + } + if *client == command.OPENSSL { + if *filePath == "" { + return fmt.Errorf("for OpenSSL commands, file path must be set") + } + } else { + if *chunkSize != 0 || *omitPayloadTrailerKey || *omitPayloadTrailer { + return fmt.Errorf("use of one or more params only suppored for OpenSSL commands") + } + } + if *client == command.CURL && *filePath != "" { + return fmt.Errorf("writing to file not currently supported for curl commands") + } return nil } diff --git a/tests/run.sh b/tests/run.sh index 0a1c723..00d7623 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -41,6 +41,7 @@ show_help() { echo " rest-create-bucket Run REST create bucket tests" echo " rest-delete-bucket-ownership-controls Run REST delete bucket ownership controls tests" echo " rest-delete-bucket-tagging Run REST delete bucket tagging tests" + echo " rest-get-bucket-location Run REST get bucket location tests" echo " rest-head-bucket Run REST head bucket tests" echo " rest-list-buckets Run REST list-buckets tests" echo " rest-multipart Run REST multipart tests" @@ -62,7 +63,7 @@ handle_param() { s3api-bucket|s3api-object|s3api-multipart|rest-base|rest-acl|rest-chunked|rest-checksum|\ rest-create-bucket|rest-head-bucket|rest-list-buckets|rest-not-implemented|\ rest-put-object|rest-versioning|rest-bucket|rest-multipart|rest-delete-bucket-ownership-controls|\ - rest-delete-bucket-tagging|setup-remove-static|rest-put-bucket-tagging) + rest-delete-bucket-tagging|setup-remove-static|rest-put-bucket-tagging|rest-get-bucket-location) run_suite "$1" ;; *) # Handle unrecognized options or positional arguments @@ -173,6 +174,8 @@ run_suite() { exit_code=1 elif ! "$HOME"/bin/bats ./tests/test_rest_delete_bucket_ownership_controls.sh; then exit_code=1 + elif ! "$HOME"/bin/bats ./tests/test_rest_get_bucket_location.sh; then + exit_code=1 elif ! "$HOME"/bin/bats ./tests/test_rest_head_bucket.sh; then exit_code=1 elif ! "$HOME"/bin/bats ./tests/test_rest_list_buckets.sh; then @@ -223,6 +226,10 @@ run_suite() { echo "Running REST delete bucket tagging tests ..." "$HOME"/bin/bats ./tests/test_rest_delete_bucket_tagging.sh || exit_code=$? ;; + rest-get-bucket-location) + echo "Running REST get bucket location tests ..." + "$HOME"/bin/bats ./tests/test_rest_get_bucket_location.sh || exit_code=$? + ;; rest-head-bucket) echo "Running REST head bucket tests ..." "$HOME"/bin/bats ./tests/test_rest_head_bucket.sh || exit_code=$? diff --git a/tests/setup_static.sh b/tests/setup_static.sh index ef4520b..958e488 100755 --- a/tests/setup_static.sh +++ b/tests/setup_static.sh @@ -17,6 +17,7 @@ source ./tests/env.sh source ./tests/util/util_object.sh source ./tests/commands/create_bucket.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh create_bucket_if_not_exists() { if ! check_param_count "create_bucket_if_not_exists" "bucket name" 1 $#; then diff --git a/tests/test_common.sh b/tests/test_common.sh index 64e8b3a..e8d970c 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -361,7 +361,7 @@ test_common_get_bucket_location() { run setup_bucket "$BUCKET_ONE_NAME" assert_success - run get_check_bucket_location "$1" "$BUCKET_ONE_NAME" + run get_check_bucket_location_various "$1" "$BUCKET_ONE_NAME" assert_success } diff --git a/tests/test_mc.sh b/tests/test_mc.sh index 12c6894..dbde75c 100755 --- a/tests/test_mc.sh +++ b/tests/test_mc.sh @@ -26,6 +26,7 @@ source ./tests/drivers/create_bucket/create_bucket_rest.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh source ./tests/drivers/get_object_tagging/get_object_tagging.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/util/util_create_bucket.sh source ./tests/util/util_head_bucket.sh diff --git a/tests/test_mc_file_count.sh b/tests/test_mc_file_count.sh index 43808ed..d05a93a 100755 --- a/tests/test_mc_file_count.sh +++ b/tests/test_mc_file_count.sh @@ -17,6 +17,7 @@ load ./bats-support/load load ./bats-assert/load +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/test_common.sh export RUN_MC=true diff --git a/tests/test_rest.sh b/tests/test_rest.sh index f2f7a95..0f1efe2 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -32,9 +32,11 @@ source ./tests/commands/put_object_tagging.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh source ./tests/drivers/copy_object/copy_object_rest.sh source ./tests/drivers/get_object_attributes/get_object_attributes_rest.sh +source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh source ./tests/drivers/get_object_tagging/get_object_tagging.sh source ./tests/drivers/get_bucket_ownership_controls/get_bucket_ownership_controls_rest.sh source ./tests/drivers/head_object/head_object_rest.sh +source ./tests/drivers/list_objects/list_objects_rest.sh source ./tests/drivers/file.sh source ./tests/drivers/xml.sh source ./tests/logger.sh @@ -306,9 +308,6 @@ test_file="test_file" } @test "REST - delete objects - no content-md5 header" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1040" - fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success bucket_name="$output" @@ -370,9 +369,6 @@ test_file="test_file" } @test "REST - PutObjectLegalHold - missing content-md5" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1311" - fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success bucket_name="$output" @@ -442,9 +438,6 @@ test_file="test_file" } @test "REST - copy object w/copy source and payload" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1242" - fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success bucket_name="$output" diff --git a/tests/test_rest_bucket.sh b/tests/test_rest_bucket.sh index 7f251ff..8d8e651 100755 --- a/tests/test_rest_bucket.sh +++ b/tests/test_rest_bucket.sh @@ -23,11 +23,12 @@ source ./tests/commands/list_buckets.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh source ./tests/drivers/get_bucket_ownership_controls/get_bucket_ownership_controls_rest.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh +source ./tests/drivers/head_bucket/head_bucket_s3api.sh source ./tests/drivers/list_buckets/list_buckets_rest.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/drivers/put_bucket_tagging/put_bucket_tagging_rest.sh source ./tests/logger.sh source ./tests/setup.sh -source ./tests/util/util_bucket.sh source ./tests/util/util_delete_object.sh source ./tests/util/util_list_buckets.sh source ./tests/util/util_lock_config.sh diff --git a/tests/test_rest_chunked.sh b/tests/test_rest_chunked.sh index 5fba1b0..214d12b 100755 --- a/tests/test_rest_chunked.sh +++ b/tests/test_rest_chunked.sh @@ -21,7 +21,8 @@ source ./tests/logger.sh source ./tests/setup.sh source ./tests/drivers/file.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh -source ./tests/util/util_bucket.sh +source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/util/util_chunked_upload.sh source ./tests/util/util_file.sh diff --git a/tests/test_rest_create_bucket.sh b/tests/test_rest_create_bucket.sh index 42da752..f2362cc 100755 --- a/tests/test_rest_create_bucket.sh +++ b/tests/test_rest_create_bucket.sh @@ -40,9 +40,6 @@ export RUN_USERS=true } @test "REST - CreateBucket w/invalid acl" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1379" - fi if [ "$RECREATE_BUCKETS" == "false" ]; then skip "skip bucket create tests for static buckets" fi @@ -124,3 +121,43 @@ export RUN_USERS=true run setup_and_create_bucket_and_check_acl "GRANT_WRITE_ACP" assert_success } + +@test "REST - CreateBucket - empty location constraint" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1644" + fi + run send_curl_command_create_bucket_expect_error "400" "InvalidLocationConstraint" "The specified location-constraint is not valid" "-locationConstraint" "" + assert_success +} + +@test "REST - CreateBucket - location constraint mismatch" { + if [ "$DIRECT" != "true" ]; then + skip "not valid for direct mode" + fi + local region="us-east-1" + if [ "$AWS_REGION" == "us-east-1" ]; then + region="us-west-1" + fi + + run send_curl_command_create_bucket_expect_error "400" "InvalidLocationConstraint" "The specified location-constraint is not valid" "-locationConstraint" "$region" + assert_success +} + +@test "REST - CreateBucket - fail - us-east-1 with 'us-east-1' location constraint" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1643" + fi + if [ "$AWS_REGION" != "us-east-1" ]; then + skip "only valid for us-east-1 region" + fi + run send_curl_command_create_bucket_expect_error "400" "InvalidLocationConstraint" "The specified location-constraint is not valid" "-locationConstraint" "us-east-1" + assert_success +} + +@test "REST - CreateBucket - location constraint error returns invalid constraint" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1645" + fi + run send_invalid_location_constraint_check_error "abc" + assert_success +} diff --git a/tests/test_rest_get_bucket_location.sh b/tests/test_rest_get_bucket_location.sh new file mode 100755 index 0000000..006a7d7 --- /dev/null +++ b/tests/test_rest_get_bucket_location.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bats + +# Copyright 2025 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/get_bucket_location/get_bucket_location_rest.sh +source ./tests/setup.sh + +@test "REST - GetBucketLocation - no bucket" { + if [ "$RECREATE_BUCKETS" == "false" ]; then + skip "not valid for static mode" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run send_rest_go_command_expect_error "404" "NoSuchBucket" "does not exist" "-bucketName" "$bucket_name" "-query" "location=" "-method" "GET" + assert_success +} + +@test "REST - GetBucketLocation - us-east-1 is returned as null" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1643" + fi + log 5 "AWS_REGION: $AWS_REGION" + if [ "$AWS_REGION" != "us-east-1" ]; then + skip "test only valid for AWS_REGION of 'us-east-1'" + fi + run create_bucket_and_run_command "$BUCKET_ONE_NAME" get_check_bucket_location "" + assert_success +} + +@test "REST - GetBucketLocation - success (non us-east-1)" { + if [ "$AWS_REGION" == "us-east-1" ]; then + skip "test not valid for us-east-1" + fi + run create_bucket_and_run_command "$BUCKET_ONE_NAME" get_check_bucket_location "$AWS_REGION" + assert_success +} diff --git a/tests/test_rest_multipart.sh b/tests/test_rest_multipart.sh index a20c0a6..466271f 100755 --- a/tests/test_rest_multipart.sh +++ b/tests/test_rest_multipart.sh @@ -174,10 +174,8 @@ test_file="test_file" run setup_bucket_and_large_file_v2 "$bucket_name" "$test_file" assert_success - run split_file "$TEST_FILE_FOLDER/$test_file" 4 - assert_success - - run upload_part_rest_without_upload_id "$bucket_name" "$test_file" + run send_openssl_go_command_expect_error "400" "InvalidArgument" "This operation does not accept partNumber without uploadId" \ + "-method" "PUT" "-bucketName" "$bucket_name" "-objectKey" "$test_file" "-query" "partNumber=1" assert_success } @@ -208,9 +206,6 @@ test_file="test_file" } @test "REST - multipart checksum w/crc64nvme, composite" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1329" - fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success bucket_name="$output" @@ -224,9 +219,6 @@ test_file="test_file" } @test "REST - multipart checksum w/sha1, full object" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1329" - fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success bucket_name="$output" diff --git a/tests/test_rest_put_object.sh b/tests/test_rest_put_object.sh index 6f75e30..33add88 100755 --- a/tests/test_rest_put_object.sh +++ b/tests/test_rest_put_object.sh @@ -214,9 +214,6 @@ export RUN_USERS=true } @test "PutObject - metadata keys are made lowercase" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1482" - fi uppercase_key="CAPITAL" uppercase_value="DUMMY" diff --git a/tests/test_s3.sh b/tests/test_s3.sh index e78d368..e244baa 100755 --- a/tests/test_s3.sh +++ b/tests/test_s3.sh @@ -17,8 +17,9 @@ load ./bats-support/load load ./bats-assert/load -source ./tests/test_common.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/util/util_file.sh +source ./tests/test_common.sh # complete-multipart-upload @test "test_complete_multipart_upload" { diff --git a/tests/test_s3_file_count.sh b/tests/test_s3_file_count.sh index 025b13d..4e50386 100755 --- a/tests/test_s3_file_count.sh +++ b/tests/test_s3_file_count.sh @@ -17,8 +17,9 @@ load ./bats-support/load load ./bats-assert/load -source ./tests/test_common.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/util/util_file.sh +source ./tests/test_common.sh @test "test_list_objects_file_count" { test_common_list_objects_file_count "s3" diff --git a/tests/test_s3api_bucket.sh b/tests/test_s3api_bucket.sh index dc8deb1..869e147 100755 --- a/tests/test_s3api_bucket.sh +++ b/tests/test_s3api_bucket.sh @@ -48,6 +48,9 @@ source ./tests/drivers/user.sh source ./tests/drivers/get_bucket_acl/get_bucket_acl.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh +source ./tests/drivers/head_bucket/head_bucket_rest.sh +source ./tests/drivers/head_bucket/head_bucket_s3api.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/util/util_create_bucket.sh source ./tests/util/util_file.sh source ./tests/util/util_head_bucket.sh diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index 878f769..72fcb5d 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -26,6 +26,7 @@ source ./tests/drivers/file.sh source ./tests/drivers/head_object/head_object_s3api.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh source ./tests/drivers/get_object_tagging/get_object_tagging.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/util/util_file.sh source ./tests/util/util_multipart.sh source ./tests/util/util_multipart_abort.sh diff --git a/tests/test_s3api_object.sh b/tests/test_s3api_object.sh index d5dd787..347d503 100755 --- a/tests/test_s3api_object.sh +++ b/tests/test_s3api_object.sh @@ -50,6 +50,7 @@ source ./tests/commands/select_object_content.sh source ./tests/drivers/copy_object/copy_object_rest.sh source ./tests/drivers/get_object_tagging/get_object_tagging.sh source ./tests/drivers/list_buckets/list_buckets_rest.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh export RUN_USERS=true diff --git a/tests/test_s3api_policy.sh b/tests/test_s3api_policy.sh index 8cf3d8c..593c9e3 100755 --- a/tests/test_s3api_policy.sh +++ b/tests/test_s3api_policy.sh @@ -36,6 +36,7 @@ source ./tests/commands/put_bucket_tagging.sh source ./tests/commands/put_object.sh source ./tests/drivers/get_bucket_acl/get_bucket_acl_s3api.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh export RUN_USERS=true diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 7dbaa1d..29d5ac3 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -20,11 +20,12 @@ load ./bats-assert/load source ./tests/setup.sh source ./tests/test_common.sh source ./tests/test_common_acl.sh -source ./tests/util/util_create_bucket.sh -source ./tests/util/util_object.sh source ./tests/commands/delete_bucket_policy.sh source ./tests/commands/get_bucket_policy.sh source ./tests/commands/put_bucket_policy.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh +source ./tests/util/util_create_bucket.sh +source ./tests/util/util_object.sh export RUN_S3CMD=true export RUN_USERS=true diff --git a/tests/test_s3cmd_file_count.sh b/tests/test_s3cmd_file_count.sh index 9efa0dc..7228489 100755 --- a/tests/test_s3cmd_file_count.sh +++ b/tests/test_s3cmd_file_count.sh @@ -17,6 +17,7 @@ load ./bats-support/load load ./bats-assert/load +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/test_common.sh export RUN_S3CMD=true diff --git a/tests/test_user_aws.sh b/tests/test_user_aws.sh index effd816..12c8eb0 100755 --- a/tests/test_user_aws.sh +++ b/tests/test_user_aws.sh @@ -20,6 +20,7 @@ load ./bats-assert/load source ./tests/test_user_common.sh source ./tests/commands/get_object.sh source ./tests/commands/put_object.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/drivers/file.sh source ./tests/drivers/user.sh diff --git a/tests/test_user_s3cmd.sh b/tests/test_user_s3cmd.sh index 2707b70..f26685c 100755 --- a/tests/test_user_s3cmd.sh +++ b/tests/test_user_s3cmd.sh @@ -17,8 +17,9 @@ load ./bats-support/load load ./bats-assert/load -source ./tests/test_user_common.sh +source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/drivers/user.sh +source ./tests/test_user_common.sh export RUN_S3CMD=true export RUN_USERS=true diff --git a/tests/util/util_bucket.sh b/tests/util/util_bucket.sh deleted file mode 100644 index 54e0cc4..0000000 --- a/tests/util/util_bucket.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env bash - -source ./tests/drivers/delete_object/delete_object_rest.sh -source ./tests/drivers/params.sh -source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh -source ./tests/drivers/list_objects/list_objects_rest.sh -source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh -source ./tests/util/util_multipart_abort.sh -source ./tests/util/util_policy.sh -source ./tests/util/util_retention.sh - -# restore bucket to pre-test state (or prep for deletion) -# param: bucket name -# return 0 on success, 1 on error -reset_bucket() { - if ! check_param_count "reset_bucket" "bucket" 1 $#; then - return 1 - fi - log 6 "reset bucket '$1'" - - if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$1"; then - log 3 "error logging bucket policy" - fi - - if ! check_object_lock_config "$1"; then - log 2 "error checking object lock config" - return 1 - fi - - if [[ "$DIRECT" != "true" ]] && ! add_governance_bypass_policy "$1"; then - log 2 "error adding governance bypass policy" - return 1 - fi - - if ! list_and_delete_objects "$1"; then - log 2 "error listing and deleting objects" - return 1 - fi - - if ! abort_all_multipart_uploads_rest "$1"; then - log 2 "error aborting all multipart uploads" - return 1 - fi - - if [ "$SKIP_ACL_TESTING" != "true" ] && ! check_ownership_rule_and_reset_acl "$1"; then - log 2 "error checking ownership rule and resetting acl" - return 1 - fi - - if ! delete_bucket_policy_rest "$1"; then - log 2 "error deleting bucket policy" - return 1 - fi - - # shellcheck disable=SC2154 - if [[ $lock_config_exists == true ]] && ! remove_retention_policy_rest "$1"; then - log 2 "error removing bucket retention policy" - return 1 - fi - - if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID"; then - log 2 "error changing bucket owner back to root" - return 1 - fi -} - -# check if bucket exists -# param: bucket name -# return 0 for true, 1 for false, 2 for error -bucket_exists() { - if ! check_param_count "bucket_exists" "bucket name" 1 $#; then - return 2 - fi - local exists=0 - head_bucket "rest" "$1" || exists=$? - log 5 "bucket exists response code: $exists" - # shellcheck disable=SC2181 - if [ $exists -eq 2 ]; then - log 2 "unexpected error checking if bucket exists" - return 2 - fi - if [ $exists -eq 0 ]; then - return 0 - fi - return 1 -} - -# params: client, bucket name(s) -# return 0 for success, 1 for failure -setup_buckets() { - if ! check_param_count_gt "minimum of 1 bucket name" 1 $#; then - return 1 - fi - for name in "$@"; do - if ! setup_bucket "$name"; then - log 2 "error setting up bucket $name" - return 1 - fi - done - return 0 -} - -# params: client, bucket name -# return 0 on successful setup, 1 on error -setup_bucket() { - log 6 "setup_bucket" - if ! check_param_count "setup_bucket" "bucket name" 1 $#; then - return 1 - fi - - bucket_exists="true" - if ! bucket_exists "$1"; then - if [[ $RECREATE_BUCKETS == "false" ]]; then - log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user" - return 1 - fi - bucket_exists="false" - fi - - if ! bucket_cleanup_if_bucket_exists "$1" "$bucket_exists"; then - log 2 "error deleting bucket or contents if they exist" - return 1 - fi - - log 5 "util.setup_bucket: bucket name: $1" - if [[ $RECREATE_BUCKETS == "true" ]]; then - if ! create_bucket "s3api" "$1"; then - log 2 "error creating bucket" - return 1 - fi - else - log 5 "skipping bucket re-creation" - fi - - if [[ $1 == "s3cmd" ]]; then - log 5 "putting bucket ownership controls" - if bucket_exists "$1" && ! put_bucket_ownership_controls "$1" "BucketOwnerPreferred"; then - log 2 "error putting bucket ownership controls" - return 1 - fi - fi - return 0 -} - -# check if bucket info can be retrieved -# param: path of bucket or folder -# return 0 for yes, 1 for no, 2 for error -bucket_is_accessible() { - if ! check_param_count "bucket_is_accessible" "bucket" 1 $#; then - return 2 - fi - local exit_code=0 - local error - error=$(aws --no-verify-ssl s3api head-bucket --bucket "$1" 2>&1) || exit_code="$?" - if [ $exit_code -eq 0 ]; then - return 0 - fi - if [[ "$error" == *"500"* ]]; then - return 1 - fi - log 2 "Error checking bucket accessibility: $error" - return 2 -} - -check_for_empty_region() { - if ! check_param_count "check_for_empty_region" "bucket" 1 $#; then - return 2 - fi - if ! head_bucket "s3api" "$BUCKET_ONE_NAME"; then - log 2 "error getting bucket info" - return 1 - fi - # shellcheck disable=SC2154 - log 5 "INFO: $bucket_info" - if ! region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1); then - log 2 "error getting region: $region" - return 1 - fi - if [[ $region == "" ]]; then - log 2 "empty bucket region" - return 1 - fi - return 0 -} diff --git a/tests/util/util_bucket_location.sh b/tests/util/util_bucket_location.sh index fe4e458..fc11afc 100644 --- a/tests/util/util_bucket_location.sh +++ b/tests/util/util_bucket_location.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -get_check_bucket_location() { +get_check_bucket_location_various() { if [ $# -ne 2 ]; then log 2 "'get_bucket_location' requires client, bucket" return 1 diff --git a/tests/util/util_object.sh b/tests/util/util_object.sh index 10faffc..5c6ab19 100644 --- a/tests/util/util_object.sh +++ b/tests/util/util_object.sh @@ -14,7 +14,6 @@ # specific language governing permissions and limitations # under the License. -source ./tests/util/util_bucket.sh source ./tests/util/util_create_bucket.sh source ./tests/util/util_mc.sh source ./tests/util/util_multipart.sh diff --git a/tests/util/util_users.sh b/tests/util/util_users.sh index 647e9ec..c4901c8 100644 --- a/tests/util/util_users.sh +++ b/tests/util/util_users.sh @@ -287,7 +287,7 @@ list_users() { list_users_versitygw() { log 6 "list_users_versitygw" - users=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users 2>&1) || local list_result=$? + users=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --region "$AWS_REGION" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users 2>&1) || local list_result=$? if [[ $list_result -ne 0 ]]; then log 2 "error listing users: $users" return 1