diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 379bf283..92502033 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -69,6 +69,7 @@ jobs: REMOVE_TEST_FILE_FOLDER: true VERSIONING_DIR: /home/tester/test-files/versioning COMMAND_LOG: /home/tester/test-files/command.log + TEST_LOG_FILE: /home/tester/test-files/test.log TIME_LOG: /home/tester/test-files/time.log PYTHON_ENV_FOLDER: /home/tester/test-files/env AUTOGENERATE_USERS: true @@ -76,6 +77,11 @@ jobs: AWS_REGION: ${{ matrix.AWS_REGION }} COVERAGE_LOG: /home/tester/test-files/coverage.log TEMPLATE_MATRIX_FILE: ${{ github.workspace }}/tests/templates/matrix.yaml + QUICK_COMPARE_SIZE: 1048576 + GENERATE_COMMAND_EXECUTABLE: /home/tester/test-files/generateCommand + WEBSITE_DOMAIN: localhost + WEBSITE: :8071 + VERSITY_LOG_FILE: /home/tester/test-files/versity.log run: | cat > /home/tester/test-files/.env <&1); then - log 2 "error getting head bucket: $result" + log 2 "error with HeadBucket command: $result" return 2 fi local callback_code=0 diff --git a/tests/commands/put_bucket_policy.sh b/tests/commands/put_bucket_policy.sh index 0b22596e..c6fb0792 100644 --- a/tests/commands/put_bucket_policy.sh +++ b/tests/commands/put_bucket_policy.sh @@ -90,3 +90,37 @@ put_bucket_policy_rest_200_or_204() { fi return 0 } + +create_website_with_random_string_and_add_permissions() { + if ! check_param_count_v2 "bucket name" 1 $#; then + return 1 + fi + local bucket_name="$1" + local response random_string policy_file + + if ! response=$(create_website_with_random_string "$bucket_name" 2>&1); then + log 2 "error creating website: $response" + return 1 + fi + read -r _ random_string <<< "$response" + + if [ "$DIRECT" == "true" ]; then + if ! put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false"; then + log 2 "error putting public access block" + return 1 + fi + fi + + if ! response=$(setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" 2>&1); then + log 2 "error setting up policy: $response" + return 1 + fi + policy_file="$response" + + if ! put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"; then + log 2 '"error putting bucket policy' + return 1 + fi + printf '%s\n' "$random_string" + return 0 +} diff --git a/tests/drivers/create_bucket/create_bucket_rest.sh b/tests/drivers/create_bucket/create_bucket_rest.sh index 45545654..988c787f 100644 --- a/tests/drivers/create_bucket/create_bucket_rest.sh +++ b/tests/drivers/create_bucket/create_bucket_rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/create_bucket.sh source ./tests/drivers/delete_bucket/delete_bucket_rest.sh source ./tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh source ./tests/drivers/get_object/get_object_rest.sh diff --git a/tests/drivers/delete_bucket/delete_bucket_rest.sh b/tests/drivers/delete_bucket/delete_bucket_rest.sh index 1d45e817..63f873b8 100644 --- a/tests/drivers/delete_bucket/delete_bucket_rest.sh +++ b/tests/drivers/delete_bucket/delete_bucket_rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/delete_bucket.sh source ./tests/commands/get_bucket_location.sh source ./tests/commands/list_buckets.sh source ./tests/drivers/get_bucket_location/get_bucket_location_rest.sh diff --git a/tests/drivers/delete_object/delete_object_rest.sh b/tests/drivers/delete_object/delete_object_rest.sh index 080124af..21610582 100644 --- a/tests/drivers/delete_object/delete_object_rest.sh +++ b/tests/drivers/delete_object/delete_object_rest.sh @@ -14,10 +14,13 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/list_object_versions.sh +source ./tests/commands/list_objects.sh source ./tests/commands/list_objects_v2.sh source ./tests/drivers/list_object_versions/list_object_versions_rest.sh source ./tests/drivers/xml.sh source ./tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh +source ./tests/util/util_object.sh list_and_delete_objects() { if ! check_param_count_v2 "bucket, lock config exists" 2 $#; then diff --git a/tests/drivers/get_bucket_website/get_bucket_website_rest.sh b/tests/drivers/get_bucket_website/get_bucket_website_rest.sh new file mode 100644 index 00000000..70a1c5d2 --- /dev/null +++ b/tests/drivers/get_bucket_website/get_bucket_website_rest.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +compare_website_suffixes() { + if ! check_param_count_v2 "data file, expected suffix" 2 $#; then + return 1 + fi + local data_file="$1" expected_suffix="$2" + + if ! check_xml_element "$data_file" "$expected_suffix" "WebsiteConfiguration" "IndexDocument" "Suffix"; then + log 2 "error checking xml element" + return 1 + fi + return 0 +} + +check_index_document_suffix() { + if ! check_param_count_v2 "bucket name, expected suffix" 2 $#; then + return 1 + fi + local bucket_name="$1" expected_suffix="$2" + + if ! send_rest_go_command_callback "200" "compare_website_suffixes" "-bucketName" "$bucket_name" "-query" "website" "--" "$expected_suffix"; then + log 2 "error sending get website command or comparing suffix" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh index 530cbef4..aef1edb2 100644 --- a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh +++ b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh @@ -14,6 +14,8 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/get_object_lock_configuration.sh + check_object_lock_config() { if ! check_param_count_v2 "bucket" 1 $#; then return 1 diff --git a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh index 479b7203..f346b14a 100644 --- a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh +++ b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh @@ -14,6 +14,8 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/get_bucket_ownership_controls.sh + put_bucket_acl_success_or_access_denied() { if ! check_param_count_v2 "bucket, acl file, username, password, expect success" 5 $#; then return 1 diff --git a/tests/drivers/put_bucket_website/put_bucket_website_rest.sh b/tests/drivers/put_bucket_website/put_bucket_website_rest.sh index a43c3aa7..7cb9ce27 100644 --- a/tests/drivers/put_bucket_website/put_bucket_website_rest.sh +++ b/tests/drivers/put_bucket_website/put_bucket_website_rest.sh @@ -24,29 +24,17 @@ create_website_with_random_string() { local bucket_name="$1" local response file_name test_string - if ! response=$(get_file_name 2>&1); then - log 2 "error getting file name: $response" - return 1 - fi - file_name="$response" - - if ! response=$(generate_random_string 8 10 2>&1); then - log 2 "error generating random string: $response" - return 1 - fi - test_string="$response" - echo "$test_string" > "$TEST_FILE_FOLDER/$file_name" - - if ! response=$(put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" 2>&1); then - log 2 "error putting random string file: $response" + if ! response=$(put_object_with_random_alnum_string "$bucket_name" 2>&1); then + log 2 "error putting object with random alnum string: $response" return 1 fi + read -r file_name test_string <<< "$response" if ! response=$(send_rest_go_command "200" "-commandType" "putBucketWebsiteConfiguration" "-bucketName" "$bucket_name" \ "-websiteConfiguration" "{\"IndexDocument\":{\"Suffix\":\"$file_name\"}}" 2>&1); then log 2 "error putting website configuration: $response" return 1 fi - printf '%s\n' "$test_string" + printf '%s\n' "$file_name $test_string" return 0 } \ No newline at end of file diff --git a/tests/drivers/put_object/put_object_rest.sh b/tests/drivers/put_object/put_object_rest.sh index e52588e3..d28ac772 100644 --- a/tests/drivers/put_object/put_object_rest.sh +++ b/tests/drivers/put_object/put_object_rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/put_object.sh source ./tests/drivers/rest.sh source ./tests/drivers/openssl.sh @@ -525,3 +526,31 @@ put_objects() { done return 0 } + +put_object_with_random_alnum_string() { + if ! check_param_count_v2 "bucket name" 1 $#; then + return 1 + fi + local bucket_name="$1" + local response file_name test_string + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! response=$(generate_random_string 8 10 2>&1); then + log 2 "error generating random string: $response" + return 1 + fi + test_string="$response" + echo "$test_string" > "$TEST_FILE_FOLDER/$file_name" + + if ! response=$(put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" 2>&1); then + log 2 "error putting random string file: $response" + return 1 + fi + echo "$file_name" "$test_string" + return 0 +} diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 0ddc194a..45722e7a 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/command.sh source ./tests/drivers/list_objects/list_objects_rest.sh source ./tests/drivers/xml.sh diff --git a/tests/drivers/user.sh b/tests/drivers/user.sh index aeb222aa..68ea4b70 100644 --- a/tests/drivers/user.sh +++ b/tests/drivers/user.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/delete_bucket_policy.sh source ./tests/drivers/delete_object/delete_object_rest.sh source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh diff --git a/tests/env.sh b/tests/env.sh index 43c1443a..566d8034 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -123,7 +123,7 @@ source_config_file() { if [ -r tests/.env ]; then source tests/.env else - echo "Warning: no .env file found in tests folder" > /dev/stderr + echo "Warning: no .env file found in tests folder" >&2 fi else # shellcheck source=./tests/.env.default diff --git a/tests/setup.sh b/tests/setup.sh index a1a3c06a..2b0672b5 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -16,6 +16,7 @@ source ./tests/env.sh source ./tests/report.sh +source ./tests/setup_common.sh source ./tests/setup_mc.sh source ./tests/drivers/delete_bucket/delete_bucket_rest.sh source ./tests/util/util_object.sh @@ -64,118 +65,25 @@ static_user_versitygw_setup() { # bats setup function setup() { - TEST_ID=$(date +"%Y%m%d-%H%M%S").$(basename "$BATS_TEST_FILENAME").$BATS_TEST_NAME - export TEST_ID + local response - source_config_file - - if ! setup_test_log_file; then - log 1 "error creating test log file" + if ! setup_env; then + log 1 "error setting up env" return 1 fi - if [ -n "$TEST_LOG_FILE" ]; then - printf "\n%s\n\n" "**** $TEST_ID ****" >> "${TEST_LOG_FILE}.${TEST_ID}" - fi - - if ! base_setup; then - log 2 "error checking env vars or running versitygw" + if ! setup_versitygw; then + log 1 "error starting versitygw app: $response" return 1 fi - if [ "$RUN_USERS" == "true" ] && [ "$SKIP_USERS_TESTS" != "true" ]; then - if ! static_user_v1_cleanup; then - log 2 "error cleaning up v1 static users" - return 1 - fi - if [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ] && [ "$AUTOGENERATE_USERS" == "false" ]; then - if ! static_user_versitygw_setup; then - log 2 "error setting up static versitygw users" - return 1 - fi - fi - fi - - log 4 "Running test $BATS_TEST_NAME" - if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then - START_TIME=$(date +%s) - export START_TIME - fi - - if [[ $RUN_S3CMD == true ]]; then - S3CMD_OPTS=() - S3CMD_OPTS+=(-c "$S3CMD_CONFIG") - S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID") - S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY") - S3CMD_OPTS+=(--region="$AWS_REGION") - export S3CMD_CONFIG S3CMD_OPTS - fi - - if [[ $RUN_MC == true ]] && ! check_add_mc_alias; then - log 1 "error checking for or adding mc alias" + if ! setup_clients; then + log 1 "error setting up clients" return 1 fi - - export AWS_PROFILE - - log 4 "********** END SETUP **********" - return 0 } # bats teardown function teardown() { - local response proc_status_one proc_status_two - - response=$(check_versity_process_status) - read -r proc_status_one proc_status_two <<< "$response" - - if [ "$proc_status_one" == "none" ] || [[ ( "$proc_status_one" == "running" ) && (( "$proc_status_two" == "none") || ( "$proc_status_two" == "running")) ]]; then - bucket_and_user_cleanup - fi - if [ "$proc_status_one" == "running" ]; then - stop_versity_process "$VERSITYGW_PID_1" - fi - if [ "$proc_status_two" == "running" ]; then - stop_versity_process "$VERSITYGW_PID_2" - fi - remove_test_file_folder_if_desired - teardown_logs -} - -static_user_v1_cleanup() { - if [ -n "$USERNAME_ONE" ]; then - if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then - log 2 "error deleting user '$USERNAME_ONE'" - return 1 - fi - fi - if [ -n "$USERNAME_TWO" ]; then - if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then - log 2 "error deleting user '$USERNAME_TWO'" - return 1 - fi - fi - return 0 -} - -bucket_and_user_cleanup() { - log 4 "********** BEGIN TEARDOWN **********" - if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then - log 3 "error cleaning up buckets after test" - fi - if [ "$SKIP_USERS_TESTS" != "true" ]; then - if ! static_user_v1_cleanup; then - log 3 "error cleaning up v1 static users" - return 1 - fi - fi - if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then - log 3 "error deleting autocreated users" - fi - if [ "$DIRECT" == "true" ] && [ -n "$distribution_created" ] && [ "$distribution_created" == "true" ]; then - if ! delete_tester_created_distributions "$USER_ID_USER_1"; then - log 3 "error deleting test-created S3 distributions" - fi - fi - return 0 + teardown_common } diff --git a/tests/setup_common.sh b/tests/setup_common.sh new file mode 100644 index 00000000..f5fbbdb0 --- /dev/null +++ b/tests/setup_common.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/env.sh +source ./tests/drivers/delete_bucket/delete_bucket_rest.sh + +setup_env() { + TEST_ID=$(date +"%Y%m%d-%H%M%S") + if [ -n "$BATS_TEST_FILENAME" ]; then + TEST_ID+=".$(basename "$BATS_TEST_FILENAME").$BATS_TEST_NAME" + fi + export TEST_ID + + source_config_file + + if ! setup_test_log_file; then + log 1 "error creating test log file" + return 1 + fi + + if [ -n "$TEST_LOG_FILE" ]; then + printf "\n%s\n\n" "**** $TEST_ID ****" >> "${TEST_LOG_FILE}.${TEST_ID}" + fi + + if [ -n "$BATS_TEST_NAME" ]; then + log 4 "Running test $BATS_TEST_NAME" + fi + if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then + START_TIME=$(date +%s) + export START_TIME + fi + + if ! check_env_vars; then + log 1 "error checking env vars" + return 1 + fi + + export AWS_PROFILE + log 4 "********** END ENV SETUP **********" + return 0 +} + +setup_versitygw() { + local params=("$@") + local response pid + + if [ "$RUN_VERSITYGW" == "true" ] && [ "$UNIT_TEST" != "true" ]; then + if ! run_versity_app "${params[@]}"; then + log 1 "error running versitygw app" + return 1 + fi + fi + + if [ "$RUN_USERS" == "true" ] && [ "$SKIP_USERS_TESTS" != "true" ]; then + if ! static_user_v1_cleanup; then + log 2 "error cleaning up v1 static users" + return 1 + fi + if [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ] && [ "$AUTOGENERATE_USERS" == "false" ]; then + if ! static_user_versitygw_setup; then + log 2 "error setting up static versitygw users" + return 1 + fi + fi + fi + log 4 "********** END VERSITYGW SETUP **********" + + return 0 +} + +setup_env_and_versitygw() { + local response + + if ! setup_env; then + log 1 "error setting up env" + return 1 + fi + if ! setup_versitygw "$@"; then + log 2 "error setting up versitygw" + return 1 + fi + return 0 +} + +setup_clients() { + if [[ $RUN_S3CMD == true ]]; then + S3CMD_OPTS=() + S3CMD_OPTS+=(-c "$S3CMD_CONFIG") + S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID") + S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY") + S3CMD_OPTS+=(--region="$AWS_REGION") + export S3CMD_CONFIG S3CMD_OPTS + fi + + if [[ $RUN_MC == true ]] && ! check_add_mc_alias; then + log 1 "error checking for or adding mc alias" + return 1 + fi + log 4 "********** END CLIENT SETUP **********" + return 0 +} + +teardown_common() { + local response proc_status_one proc_status_two + + response=$(check_versity_process_status) + read -r proc_status_one proc_status_two <<< "$response" + + log 5 "proc status one: '$proc_status_one'" + log 5 "proc status two: '$proc_status_two'" + if [ "$proc_status_one" == "none" ] || [[ ( "$proc_status_one" == "running" ) && (( "$proc_status_two" == "none") || ( "$proc_status_two" == "running")) ]]; then + if ! bucket_and_user_cleanup; then + log 3 "bucket and user cleanup not properly done" + fi + fi + if [ "$proc_status_one" == "running" ]; then + if ! stop_versity_process "$VERSITYGW_PID_1"; then + log 3 "unable to properly stop versitygw process" + fi + fi + if [ "$proc_status_two" == "running" ]; then + if ! stop_versity_process "$VERSITYGW_PID_2"; then + log 3 "unable to properly stop second versity process" + fi + fi + if ! remove_test_file_folder_if_desired; then + log 3 "test file folder cleanup error" + fi + if ! teardown_logs; then + log 3 "log file teardown error" + fi + return 0 +} + +static_user_v1_cleanup() { + if [ -n "$USERNAME_ONE" ]; then + if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then + log 2 "error deleting user '$USERNAME_ONE'" + return 1 + fi + fi + if [ -n "$USERNAME_TWO" ]; then + if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then + log 2 "error deleting user '$USERNAME_TWO'" + return 1 + fi + fi + return 0 +} + +bucket_and_user_cleanup() { + log 4 "********** BEGIN TEARDOWN **********" + if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then + log 3 "error cleaning up buckets after test" + fi + if [ "$SKIP_USERS_TESTS" != "true" ] && [ "$RUN_USERS" == "true" ]; then + if ! static_user_v1_cleanup; then + log 3 "error cleaning up v1 static users" + return 1 + fi + if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then + log 3 "error deleting autocreated users" + fi + fi + if [ "$DIRECT" == "true" ] && [ -n "$distribution_created" ] && [ "$distribution_created" == "true" ]; then + if ! delete_tester_created_distributions "$USER_ID_USER_1"; then + log 3 "error deleting test-created S3 distributions" + fi + fi + return 0 +} diff --git a/tests/setup_static.sh b/tests/setup_static.sh index 8c6857ae..cdaa1c73 100755 --- a/tests/setup_static.sh +++ b/tests/setup_static.sh @@ -14,7 +14,8 @@ # specific language governing permissions and limitations # under the License. -source ./tests/env.sh +source ./tests/logger.sh +source ./tests/setup_common.sh source ./tests/util/util_object.sh source ./tests/commands/create_bucket.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh @@ -40,7 +41,11 @@ create_bucket_if_not_exists() { return 0 } -base_setup +if ! setup_env_and_versitygw; then + log 1 "error setting up env and/or versitygw" + exit 1 +fi + if ! create_bucket_if_not_exists "$BUCKET_ONE_NAME"; then log 2 "error creating static bucket one" elif ! create_bucket_if_not_exists "$BUCKET_TWO_NAME"; then @@ -49,6 +54,6 @@ fi # shellcheck disable=SC2034 RECREATE_BUCKETS=false -if ! stop_versity_process "$VERSITYGW_PID_1"; then - log 2 "error stopping versity" +if ! teardown_common; then + log 2 "teardown error" fi diff --git a/tests/test_rest_list_buckets_s3_posix.sh b/tests/test_rest_list_buckets_s3_posix.sh new file mode 100755 index 00000000..8ab4c2d7 --- /dev/null +++ b/tests/test_rest_list_buckets_s3_posix.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bats + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/commands/list_buckets.sh +source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/list_buckets/list_buckets_rest.sh +source ./tests/logger.sh +source ./tests/setup.sh + +setup() { + if ! setup_env; then + log 1 "error setting up env" + return 1 + fi + if ! BACKEND=s3ToPosix setup_versitygw; then + log 1 "error setting up s3->posix versitygw config" + return 1 + fi + return 0 +} + +teardown() { + BACKEND=s3ToPosix teardown_common +} + +@test "ListBuckets - s3 -> posix" { + skip "https://github.com/versity/versitygw/issues/2309" + + local bucket_name + + run setup_bucket_v3 "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run list_check_buckets_rest "$bucket_name" + assert_success +} \ No newline at end of file diff --git a/tests/test_rest_website.sh b/tests/test_rest_website.sh index 3b82effb..c309f9fb 100755 --- a/tests/test_rest_website.sh +++ b/tests/test_rest_website.sh @@ -19,14 +19,40 @@ load ./bats-assert/load source ./tests/commands/put_public_access_block.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/get_bucket_website/get_bucket_website_rest.sh source ./tests/drivers/put_bucket_website/put_bucket_website_rest.sh source ./tests/drivers/cloudfront.sh source ./tests/drivers/string.sh -source ./tests/setup.sh +source ./tests/setup_common.sh + +setup() { + if ! setup_env; then + log 1 "error setting up env" + return 1 + fi +} + +setup_versitygw_for_website() { + optional_params=("$@") + + run setup_versitygw "${optional_params[@]}" + assert_success + read -r process_id process_id_two <<< "$output" + export VERSITYGW_PID_1="$process_id" + if [ -n "$process_id_two" ]; then + export VERSITYGW_PID_2="$process_id_two" + fi +} + +teardown() { + teardown_common +} @test "PutBucketWebsite - empty payload" { local bucket_name + setup_versitygw_for_website + run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success bucket_name="$output" @@ -41,6 +67,8 @@ source ./tests/setup.sh fi local bucket_name + setup_versitygw_for_website + run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success bucket_name="$output" @@ -56,27 +84,17 @@ source ./tests/setup.sh fi distribution_created=false - local bucket_name policy_file distribution_domain http_domain + setup_versitygw_for_website + + local bucket_name distribution_domain http_domain run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success bucket_name="$output" - run create_website_with_random_string "$bucket_name" - assert_success - random_string="$output" - - if [ "$DIRECT" == "true" ]; then - run put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false" - assert_success - fi - - run setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" - assert_success - policy_file="$output" - - run put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file" + run create_website_with_random_string_and_add_permissions "$bucket_name" assert_success + read -r random_string <<< "$output" if [ "$DIRECT" == "true" ]; then run create_cloudfront_distribution "$bucket_name" "index.html" "${bucket_name}.s3-website.us-east-1.amazonaws.com" @@ -95,7 +113,27 @@ source ./tests/setup.sh } @test "PutBucketWebsite - IndexDocument suffix" { - local bucket_name policy_file random_string + local bucket_name random_string + + setup_versitygw_for_website + + run setup_bucket_v3 "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run create_website_with_random_string_and_add_permissions "$bucket_name" + assert_success + read -r random_string <<< "$output" + + run curl -ks "https://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}" + assert_success + assert_output "$random_string" +} + +@test "REST - GetBucketWebsite - IndexDocument Suffix, DeleteBucketWebsite" { + local bucket_name test_file random_string + + setup_versitygw_for_website run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success @@ -103,21 +141,33 @@ source ./tests/setup.sh run create_website_with_random_string "$bucket_name" assert_success - random_string="$output" + read -r test_file random_string <<< "$output" - if [ "$DIRECT" == "true" ]; then - run put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false" - assert_success - fi - - run setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" - assert_success - policy_file="$output" - - run put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file" + run check_index_document_suffix "$bucket_name" "$test_file" assert_success - run curl -ks "https://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}" + run send_rest_go_command "204" "-method" "DELETE" "-query" "website" "-bucketName" "$bucket_name" + assert_success + + run send_rest_go_command_expect_error_with_specific_arg_name_value "404" "NoSuchWebsiteConfiguration" \ + "does not have a website configuration" "BucketName" "$bucket_name" "-query" "website" "-bucketName" "$bucket_name" + assert_success +} + +@test "REST - GetBucketWebsite - no HTTPS" { + local bucket_name random_string + + setup_versitygw_for_website "--website-no-tls" + + run setup_bucket_v3 "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run create_website_with_random_string_and_add_permissions "$bucket_name" + assert_success + read -r random_string <<< "$output" + + run curl -ks "http://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}" assert_success assert_output "$random_string" } diff --git a/tests/util/util_multipart_abort.sh b/tests/util/util_multipart_abort.sh index 7ab97612..0af2604e 100644 --- a/tests/util/util_multipart_abort.sh +++ b/tests/util/util_multipart_abort.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/list_multipart_uploads.sh source ./tests/util/util_multipart_before_completion.sh check_abort_access_denied() { diff --git a/tests/util/util_policy.sh b/tests/util/util_policy.sh index 62a50f1e..f3a8675a 100644 --- a/tests/util/util_policy.sh +++ b/tests/util/util_policy.sh @@ -14,6 +14,8 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/get_bucket_policy.sh + check_for_empty_policy() { if ! check_param_count "check_for_empty_policy" "command type, bucket name" 2 $#; then return 1 diff --git a/tests/util/util_retention.sh b/tests/util/util_retention.sh index 708dfe8a..028a5f48 100644 --- a/tests/util/util_retention.sh +++ b/tests/util/util_retention.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/put_bucket_policy.sh source ./tests/drivers/params.sh # params: bucket name diff --git a/tests/versity.sh b/tests/versity.sh index bca9df91..329286b4 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -29,6 +29,7 @@ start_versity_process() { process_id="$response" printf -v "VERSITYGW_PID_$1" '%s' "$process_id" + log 4 "versitygw PID for $1: $process_id" export VERSITYGW_PID_"$1" return 0 @@ -38,9 +39,10 @@ build_run_and_log_command() { if ! check_param_count_gt "versitygw process number (1 or 2), command array" 2 $#; then return 1 fi + local process_number="$1" command_array=("${@:2}") local response full_command versitygw_log_file_name="" pid check_result - IFS=' ' read -r -a full_command <<< "${@:2}" + IFS=' ' read -r -a full_command <<< "${command_array[@]}" log 5 "versity command: ${full_command[*]}" if [ -n "$COMMAND_LOG" ]; then if ! response=$(mask_args "${full_command[*]}" 2>&1); then @@ -51,15 +53,15 @@ build_run_and_log_command() { echo "$response" >> "$COMMAND_LOG" fi if [ -n "$VERSITY_LOG_FILE" ]; then - versitygw_log_file_name="$VERSITY_LOG_FILE.$TEST_ID".$1 - echo "****************************** VERSITYGW $1 LOG ***********************************" >> "$versitygw_log_file_name" + versitygw_log_file_name="$VERSITY_LOG_FILE.$TEST_ID".$process_number + printf '****************************** VERSITYGW %s LOG \***********************************\n' "$1" >> "$versitygw_log_file_name" "${full_command[@]}" >> "$versitygw_log_file_name" 2>&1 & else "${full_command[@]}" >/dev/null 2>&1 & fi pid="$!" - if ! verify_process_started "$pid" "$versitygw_log_file_name"; then + if ! verify_process_started "$process_number" "$pid" "$versitygw_log_file_name"; then return 1 fi @@ -67,20 +69,37 @@ build_run_and_log_command() { return 0 } +get_app_two_endpoint_url() { + local endpoint="http://localhost" + + if [ -n "$PORT_TWO" ]; then + endpoint+="$PORT_TWO" + else + endpoint+=":7071" + fi + printf '%s\n' "$endpoint" + return 0 +} + verify_process_started() { - if ! check_param_count_ge_le "pid, log file (if any)" 1 2 $#; then + if ! check_param_count_ge_le "app ID, pid, log file (if any)" 2 3 $#; then return 1 fi - local check_result process_running="false" proc_state + local app_id="$1" pid="$2" log_file="$3" + local check_result process_running="false" proc_state params endpoint_url for ((check_num=1; check_num<=3; check_num++)); do sleep 1 - if [ "$process_running" == "false" ] && check_result=$(kill -0 "$1" 2>&1); then + if [ "$process_running" == "false" ] && check_result=$(kill -0 "$pid" 2>&1); then process_running="true" fi if [ "$process_running" == "true" ]; then - proc_state=$(ps -p "$1" -o state= 2>/dev/null | tr -d ' ') - if [ "$proc_state" != "Z" ] && list_buckets_rest "" "" >/dev/null; then + proc_state=$(ps -p "$pid" -o state= 2>/dev/null | tr -d ' ') + if [ "$app_id" == 2 ]; then + endpoint_url="$(get_app_two_endpoint_url)" + params="AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_TWO AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_TWO AWS_ENDPOINT_URL=$endpoint_url" + fi + if [ "$proc_state" != "Z" ] && list_buckets_rest "$params" "" >/dev/null; then return 0 fi fi @@ -93,27 +112,39 @@ verify_process_started() { else log 1 "process running in state '$proc_state', but not communicating properly" fi - if [[ -n "$2" ]]; then - log 1 "log data: '$(cat "$2")'" + if [[ -n "$log_file" ]]; then + log 1 "log data: '$(cat "$log_file")'" fi return 1 } run_versity_app_posix() { - if ! check_param_count "run_versity_app_posix" "access ID, secret key, versityid app index" 3 $#; then + if ! check_param_count_gt "access ID, secret key, versitygw app index, optional params" 3 $#; then return 1 fi - base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION") + local access_id="$1" secret_key="$2" versitygw_app_index="$3" optional_params=("${@:4}") + local -a base_command + local response process_id + + base_command=("$VERSITY_EXE" --access="$access_id" --secret="$secret_key" --region="$AWS_REGION") if [ -n "$RUN_USERS" ]; then # shellcheck disable=SC2153 IFS=' ' read -r -a iam_array <<< "$IAM_PARAMS" fi base_command+=("${iam_array[@]}") - if [ -n "$CERT" ] && [ -n "$KEY" ]; then - base_command+=(--cert "$CERT" --key "$KEY") - fi - if [ -n "$PORT" ]; then - base_command+=(--port ":$PORT") + if [ "$BACKEND" == "s3ToPosix" ]; then + if [ -n "$PORT_TWO" ]; then + base_command+=(--port ":$PORT_TWO") + else + base_command+=(--port ":7071") + fi + else + if [ -n "$CERT" ] && [ -n "$KEY" ]; then + base_command+=(--cert "$CERT" --key "$KEY") + fi + if [ -n "$PORT" ]; then + base_command+=(--port ":$PORT") + fi fi if [ -n "$WEBSITE" ]; then base_command+=(--website "$WEBSITE") @@ -121,13 +152,15 @@ run_versity_app_posix() { if [ -n "$WEBSITE_DOMAIN" ]; then base_command+=(--website-domain "$WEBSITE_DOMAIN") fi + base_command+=("${optional_params[@]}") base_command+=(posix) if [ -n "$VERSIONING_DIR" ]; then base_command+=(--versioning-dir "$VERSIONING_DIR") fi base_command+=("$LOCAL_FOLDER") + log 5 "base command: ${base_command[*]}" - if ! start_versity_process "$3" "${base_command[@]}"; then + if ! start_versity_process "$versitygw_app_index" "${base_command[@]}"; then log 1 "error starting versity process" return 1 fi @@ -138,7 +171,11 @@ run_versity_app_scoutfs() { if ! check_param_count "run_versity_app_scoutfs" "access ID, secret key, versityid app index" 3 $#; then return 1 fi - base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION" --iam-dir="$USERS_FOLDER") + local access_id="$1" secret_key="$2" app_index="$3" + local -a base_command + local response process_id + + base_command=("$VERSITY_EXE" --access="$access_id" --secret="$secret_key" --region="$AWS_REGION" --iam-dir="$USERS_FOLDER") if [ -n "$CERT" ] && [ -n "$KEY" ]; then base_command+=(--cert "$CERT" --key "$KEY") fi @@ -147,29 +184,36 @@ run_versity_app_scoutfs() { fi base_command+=(scoutfs "$LOCAL_FOLDER") - if ! start_versity_process "$3" "${base_command[@]}"; then - log 1 "error starting versity process" + if ! start_versity_process "$app_index" "${base_command[@]}"; then + log 1 "error starting versity process: $response" return 1 fi return 0 } run_versity_app_s3() { - if ! check_param_count "run_versity_app_s3" "versityid app index" 1 $#; then + if ! check_param_count "run_versity_app_s3" "server key ID, server access key, dest key ID, dest access key, app index" 5 $#; then return 1 fi - base_command=("$VERSITY_EXE" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY" --region="$AWS_REGION") + local server_key_id="$1" server_access_key="$2" dest_key_id="$3" dest_access_key="$4" app_index="$5" + local -a base_command + local endpoint_url + + base_command=("$VERSITY_EXE" --access="$server_key_id" --secret="$server_access_key" --region="$AWS_REGION") if [ -n "$CERT" ] && [ -n "$KEY" ]; then base_command+=(--cert "$CERT" --key "$KEY") fi - if [ -n "$PORT_TWO" ]; then - base_command+=(--port ":$PORT_TWO") - else - base_command+=(--port ":7071") + if [ -n "$PORT" ]; then + base_command+=(--port ":$PORT") fi - base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com) + if [ "$BACKEND" == "s3ToPosix" ]; then + endpoint_url=$(get_app_two_endpoint_url) + else + endpoint_url="$S3_AWS_ENDPOINT_URL" + fi + base_command+=(s3 --access="$dest_key_id" --secret="$dest_access_key" --region="$AWS_REGION" --endpoint="$endpoint_url") - if ! start_versity_process "$1" "${base_command[@]}"; then + if ! start_versity_process "$app_index" "${base_command[@]}"; then log 2 "error starting versity process" return 1 fi @@ -177,29 +221,40 @@ run_versity_app_s3() { } run_versity_app() { + local additional_params=("$@") + local response process_id + if [[ $BACKEND == 'posix' ]]; then - if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then + if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" "${additional_params[@]}"; then + log 2 "error running POSIX versity app: $response" return 1 fi elif [[ $BACKEND == 'scoutfs' ]]; then if ! run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then + log 2 "error running scoutfs versity app: $response" return 1 fi - elif [[ $BACKEND == 's3' ]]; then - if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then - return 1 + elif [[ $BACKEND == 's3' ]] || [ "$BACKEND" == "s3ToPosix" ]; then + # start in reverse order to check with ListBuckets for both + if [ "$BACKEND" == 's3ToPosix' ]; then + if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID_TWO" "$AWS_SECRET_ACCESS_KEY_TWO" "2"; then + log 2 "error running versitygw with posix backend in s3 config" + return 1 + fi fi - if ! run_versity_app_s3 "2"; then + if ! run_versity_app_s3 "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_ACCESS_KEY_ID_TWO" "$AWS_SECRET_ACCESS_KEY_TWO" "1"; then + log 2 "error running versitygw with s3 backend in s3 config" return 1 fi else log 1 "unrecognized backend type $BACKEND" return 1 fi + printf '%s %s\n' "$VERSITYGW_PID_1" "$VERSITYGW_PID_2" if [[ $IAM_TYPE != "s3" ]]; then return 0 fi - if bucket_exists "$USERS_BUCKET"; then + if bucket_exists "$USERS_BUCKET" >/dev/null; then return 0 fi if ! create_bucket "s3api" "$USERS_BUCKET"; then @@ -230,13 +285,13 @@ check_versity_process_status() { status_one="none" status_two="none" if [ "$RUN_VERSITYGW" == "true" ]; then - if [[ -n "$VERSITYGW_PID_1" ]] && verify_process_started "$VERSITYGW_PID_1" >/dev/null; then + if [[ -n "$VERSITYGW_PID_1" ]] && verify_process_started "1" "$VERSITYGW_PID_1" >/dev/null; then status_one="running" else status_one="failed" fi - if [ "$BACKEND" == "s3" ]; then - if [ -n "$VERSITYGW_PID_2" ] && verify_process_started "$VERSITYGW_PID_2" >/dev/null; then + if [ "$BACKEND" == "s3ToPosix" ]; then + if [ -n "$VERSITYGW_PID_2" ] && verify_process_started "2" "$VERSITYGW_PID_2" >/dev/null; then status_two="running" else status_two="failed"