diff --git a/tests/commands/put_bucket_policy.sh b/tests/commands/put_bucket_policy.sh index 0b22596e..c6fb0792 100644 --- a/tests/commands/put_bucket_policy.sh +++ b/tests/commands/put_bucket_policy.sh @@ -90,3 +90,37 @@ put_bucket_policy_rest_200_or_204() { fi return 0 } + +create_website_with_random_string_and_add_permissions() { + if ! check_param_count_v2 "bucket name" 1 $#; then + return 1 + fi + local bucket_name="$1" + local response random_string policy_file + + if ! response=$(create_website_with_random_string "$bucket_name" 2>&1); then + log 2 "error creating website: $response" + return 1 + fi + read -r _ random_string <<< "$response" + + if [ "$DIRECT" == "true" ]; then + if ! put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false"; then + log 2 "error putting public access block" + return 1 + fi + fi + + if ! response=$(setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" 2>&1); then + log 2 "error setting up policy: $response" + return 1 + fi + policy_file="$response" + + if ! put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"; then + log 2 '"error putting bucket policy' + return 1 + fi + printf '%s\n' "$random_string" + return 0 +} diff --git a/tests/drivers/create_bucket/create_bucket_rest.sh b/tests/drivers/create_bucket/create_bucket_rest.sh index 45545654..988c787f 100644 --- a/tests/drivers/create_bucket/create_bucket_rest.sh +++ b/tests/drivers/create_bucket/create_bucket_rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/create_bucket.sh source ./tests/drivers/delete_bucket/delete_bucket_rest.sh source ./tests/drivers/get_bucket_acl/get_bucket_acl_rest.sh source ./tests/drivers/get_object/get_object_rest.sh diff --git a/tests/drivers/delete_bucket/delete_bucket_rest.sh b/tests/drivers/delete_bucket/delete_bucket_rest.sh index 1d45e817..63f873b8 100644 --- a/tests/drivers/delete_bucket/delete_bucket_rest.sh +++ b/tests/drivers/delete_bucket/delete_bucket_rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/delete_bucket.sh source ./tests/commands/get_bucket_location.sh source ./tests/commands/list_buckets.sh source ./tests/drivers/get_bucket_location/get_bucket_location_rest.sh diff --git a/tests/drivers/delete_object/delete_object_rest.sh b/tests/drivers/delete_object/delete_object_rest.sh index 080124af..21610582 100644 --- a/tests/drivers/delete_object/delete_object_rest.sh +++ b/tests/drivers/delete_object/delete_object_rest.sh @@ -14,10 +14,13 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/list_object_versions.sh +source ./tests/commands/list_objects.sh source ./tests/commands/list_objects_v2.sh source ./tests/drivers/list_object_versions/list_object_versions_rest.sh source ./tests/drivers/xml.sh source ./tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh +source ./tests/util/util_object.sh list_and_delete_objects() { if ! check_param_count_v2 "bucket, lock config exists" 2 $#; then diff --git a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh index 530cbef4..aef1edb2 100644 --- a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh +++ b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh @@ -14,6 +14,8 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/get_object_lock_configuration.sh + check_object_lock_config() { if ! check_param_count_v2 "bucket" 1 $#; then return 1 diff --git a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh index 479b7203..f346b14a 100644 --- a/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh +++ b/tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh @@ -14,6 +14,8 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/get_bucket_ownership_controls.sh + put_bucket_acl_success_or_access_denied() { if ! check_param_count_v2 "bucket, acl file, username, password, expect success" 5 $#; then return 1 diff --git a/tests/drivers/put_object/put_object_rest.sh b/tests/drivers/put_object/put_object_rest.sh index 4a322b77..d28ac772 100644 --- a/tests/drivers/put_object/put_object_rest.sh +++ b/tests/drivers/put_object/put_object_rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/put_object.sh source ./tests/drivers/rest.sh source ./tests/drivers/openssl.sh diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 0ddc194a..45722e7a 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/command.sh source ./tests/drivers/list_objects/list_objects_rest.sh source ./tests/drivers/xml.sh diff --git a/tests/drivers/user.sh b/tests/drivers/user.sh index aeb222aa..68ea4b70 100644 --- a/tests/drivers/user.sh +++ b/tests/drivers/user.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/delete_bucket_policy.sh source ./tests/drivers/delete_object/delete_object_rest.sh source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh source ./tests/drivers/put_bucket_acl/put_bucket_acl_rest.sh diff --git a/tests/setup.sh b/tests/setup.sh index a1a3c06a..27b12561 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -16,6 +16,7 @@ source ./tests/env.sh source ./tests/report.sh +source ./tests/setup_env_and_versitygw.sh source ./tests/setup_mc.sh source ./tests/drivers/delete_bucket/delete_bucket_rest.sh source ./tests/util/util_object.sh @@ -124,58 +125,5 @@ setup() { # bats teardown function teardown() { - local response proc_status_one proc_status_two - - response=$(check_versity_process_status) - read -r proc_status_one proc_status_two <<< "$response" - - if [ "$proc_status_one" == "none" ] || [[ ( "$proc_status_one" == "running" ) && (( "$proc_status_two" == "none") || ( "$proc_status_two" == "running")) ]]; then - bucket_and_user_cleanup - fi - if [ "$proc_status_one" == "running" ]; then - stop_versity_process "$VERSITYGW_PID_1" - fi - if [ "$proc_status_two" == "running" ]; then - stop_versity_process "$VERSITYGW_PID_2" - fi - remove_test_file_folder_if_desired - teardown_logs -} - -static_user_v1_cleanup() { - if [ -n "$USERNAME_ONE" ]; then - if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then - log 2 "error deleting user '$USERNAME_ONE'" - return 1 - fi - fi - if [ -n "$USERNAME_TWO" ]; then - if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then - log 2 "error deleting user '$USERNAME_TWO'" - return 1 - fi - fi - return 0 -} - -bucket_and_user_cleanup() { - log 4 "********** BEGIN TEARDOWN **********" - if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then - log 3 "error cleaning up buckets after test" - fi - if [ "$SKIP_USERS_TESTS" != "true" ]; then - if ! static_user_v1_cleanup; then - log 3 "error cleaning up v1 static users" - return 1 - fi - fi - if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then - log 3 "error deleting autocreated users" - fi - if [ "$DIRECT" == "true" ] && [ -n "$distribution_created" ] && [ "$distribution_created" == "true" ]; then - if ! delete_tester_created_distributions "$USER_ID_USER_1"; then - log 3 "error deleting test-created S3 distributions" - fi - fi - return 0 + teardown_common } diff --git a/tests/setup_env_and_versitygw.sh b/tests/setup_env_and_versitygw.sh new file mode 100644 index 00000000..52f3ded7 --- /dev/null +++ b/tests/setup_env_and_versitygw.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/env.sh + +setup_env() { + TEST_ID=$(date +"%Y%m%d-%H%M%S").$(basename "$BATS_TEST_FILENAME").$BATS_TEST_NAME + export TEST_ID + + source_config_file + + if ! setup_test_log_file; then + log 1 "error creating test log file" + return 1 + fi + + if [ -n "$TEST_LOG_FILE" ]; then + printf "\n%s\n\n" "**** $TEST_ID ****" >> "${TEST_LOG_FILE}.${TEST_ID}" + fi + + log 4 "Running test $BATS_TEST_NAME" + if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then + START_TIME=$(date +%s) + export START_TIME + fi + + if ! check_env_vars; then + log 1 "error checking env vars" + return 1 + fi + + export AWS_PROFILE + log 4 "********** END ENV SETUP **********" + return 0 +} + +setup_versitygw() { + local params=("$@") + local response + + if [ "$RUN_VERSITYGW" == "true" ] && [ "$UNIT_TEST" != "true" ]; then + if ! response=$(run_versity_app "${params[@]}" 2>&1); then + log 1 "error running versitygw app: $response" + return 1 + fi + printf '%s\n' "$response" + fi + log 4 "********** END VERSITYGW SETUP **********" + return 0 +} + +setup_clients() { + if [[ $RUN_S3CMD == true ]]; then + S3CMD_OPTS=() + S3CMD_OPTS+=(-c "$S3CMD_CONFIG") + S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID") + S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY") + S3CMD_OPTS+=(--region="$AWS_REGION") + export S3CMD_CONFIG S3CMD_OPTS + fi + + if [[ $RUN_MC == true ]] && ! check_add_mc_alias; then + log 1 "error checking for or adding mc alias" + return 1 + fi + log 4 "********** END CLIENT SETUP **********" + return 0 +} + +teardown_common() { + local response proc_status_one proc_status_two + + response=$(check_versity_process_status) + read -r proc_status_one proc_status_two <<< "$response" + + log 5 "proc status one: '$proc_status_one'" + if [ "$proc_status_one" == "none" ] || [[ ( "$proc_status_one" == "running" ) && (( "$proc_status_two" == "none") || ( "$proc_status_two" == "running")) ]]; then + bucket_and_user_cleanup + fi + if [ "$proc_status_one" == "running" ]; then + stop_versity_process "$VERSITYGW_PID_1" + fi + if [ "$proc_status_two" == "running" ]; then + stop_versity_process "$VERSITYGW_PID_2" + fi + remove_test_file_folder_if_desired + teardown_logs +} + +static_user_v1_cleanup() { + if [ -n "$USERNAME_ONE" ]; then + if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then + log 2 "error deleting user '$USERNAME_ONE'" + return 1 + fi + fi + if [ -n "$USERNAME_TWO" ]; then + if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then + log 2 "error deleting user '$USERNAME_TWO'" + return 1 + fi + fi + return 0 +} + +bucket_and_user_cleanup() { + log 4 "********** BEGIN TEARDOWN **********" + if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then + log 3 "error cleaning up buckets after test" + fi + if [ "$SKIP_USERS_TESTS" != "true" ] && [ "$RUN_USERS" == "true" ]; then + if ! static_user_v1_cleanup; then + log 3 "error cleaning up v1 static users" + return 1 + fi + if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then + log 3 "error deleting autocreated users" + fi + fi + if [ "$DIRECT" == "true" ] && [ -n "$distribution_created" ] && [ "$distribution_created" == "true" ]; then + if ! delete_tester_created_distributions "$USER_ID_USER_1"; then + log 3 "error deleting test-created S3 distributions" + fi + fi + return 0 +} diff --git a/tests/test_rest_website.sh b/tests/test_rest_website.sh index 3bff45c5..d058bbda 100755 --- a/tests/test_rest_website.sh +++ b/tests/test_rest_website.sh @@ -23,11 +23,26 @@ source ./tests/drivers/get_bucket_website/get_bucket_website_rest.sh source ./tests/drivers/put_bucket_website/put_bucket_website_rest.sh source ./tests/drivers/cloudfront.sh source ./tests/drivers/string.sh -source ./tests/setup.sh +source ./tests/setup_env_and_versitygw.sh +#source ./tests/setup.sh + +setup() { + if ! setup_env; then + log 1 "error setting up env" + return 1 + fi +} + +teardown() { + teardown_common +} @test "PutBucketWebsite - empty payload" { local bucket_name + run setup_versitygw + assert_success + run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success bucket_name="$output" @@ -42,6 +57,9 @@ source ./tests/setup.sh fi local bucket_name + run setup_versitygw + assert_success + run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success bucket_name="$output" @@ -57,6 +75,9 @@ source ./tests/setup.sh fi distribution_created=false + run setup_versitygw + assert_success + local bucket_name policy_file distribution_domain http_domain run setup_bucket_v3 "$BUCKET_ONE_NAME" @@ -98,25 +119,18 @@ source ./tests/setup.sh @test "PutBucketWebsite - IndexDocument suffix" { local bucket_name policy_file random_string + run setup_versitygw + assert_success + process_id="$output" + export VERSITYGW_PID_1="$process_id" + run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success bucket_name="$output" - run create_website_with_random_string "$bucket_name" - assert_success - read -r _ random_string <<< "$output" - - if [ "$DIRECT" == "true" ]; then - run put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false" - assert_success - fi - - run setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" - assert_success - policy_file="$output" - - run put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file" + run create_website_with_random_string_and_add_permissions "$bucket_name" assert_success + read -r random_string <<< "$output" run curl -ks "https://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}" assert_success @@ -126,6 +140,11 @@ source ./tests/setup.sh @test "REST - GetBucketWebsite - IndexDocument Suffix, DeleteBucketWebsite" { local bucket_name test_file random_string + run setup_versitygw + assert_success + process_id="$output" + export VERSITYGW_PID_1="$process_id" + run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success bucket_name="$output" @@ -146,5 +165,21 @@ source ./tests/setup.sh } @test "REST - GetBucketWebsite - no HTTPS" { + run setup_versitygw "--website-no-tls" + assert_success + process_id="$output" + log 5 "process ID: $process_id" + export VERSITYGW_PID_1="$process_id" + run setup_bucket_v3 "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run create_website_with_random_string_and_add_permissions "$bucket_name" + assert_success + read -r random_string <<< "$output" + + run curl -ks "http://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}" + assert_success + assert_output "$random_string" } diff --git a/tests/util/util_multipart_abort.sh b/tests/util/util_multipart_abort.sh index 7ab97612..0af2604e 100644 --- a/tests/util/util_multipart_abort.sh +++ b/tests/util/util_multipart_abort.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/list_multipart_uploads.sh source ./tests/util/util_multipart_before_completion.sh check_abort_access_denied() { diff --git a/tests/util/util_policy.sh b/tests/util/util_policy.sh index 62a50f1e..f3a8675a 100644 --- a/tests/util/util_policy.sh +++ b/tests/util/util_policy.sh @@ -14,6 +14,8 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/get_bucket_policy.sh + check_for_empty_policy() { if ! check_param_count "check_for_empty_policy" "command type, bucket name" 2 $#; then return 1 diff --git a/tests/util/util_retention.sh b/tests/util/util_retention.sh index 708dfe8a..028a5f48 100644 --- a/tests/util/util_retention.sh +++ b/tests/util/util_retention.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/put_bucket_policy.sh source ./tests/drivers/params.sh # params: bucket name diff --git a/tests/versity.sh b/tests/versity.sh index bca9df91..bb494862 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -29,8 +29,10 @@ start_versity_process() { process_id="$response" printf -v "VERSITYGW_PID_$1" '%s' "$process_id" + log 4 "versitygw PID for $1: $process_id" export VERSITYGW_PID_"$1" + printf '%s\n' "$process_id" return 0 } @@ -52,7 +54,7 @@ build_run_and_log_command() { fi if [ -n "$VERSITY_LOG_FILE" ]; then versitygw_log_file_name="$VERSITY_LOG_FILE.$TEST_ID".$1 - echo "****************************** VERSITYGW $1 LOG ***********************************" >> "$versitygw_log_file_name" + printf '****************************** VERSITYGW %s LOG \***********************************\n' "$1" >> "$versitygw_log_file_name" "${full_command[@]}" >> "$versitygw_log_file_name" 2>&1 & else "${full_command[@]}" >/dev/null 2>&1 & @@ -100,10 +102,14 @@ verify_process_started() { } run_versity_app_posix() { - if ! check_param_count "run_versity_app_posix" "access ID, secret key, versityid app index" 3 $#; then + if ! check_param_count_gt "access ID, secret key, versitygw app index, optional params" 3 $#; then return 1 fi - base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION") + local access_id="$1" secret_key="$2" versitygw_app_index="$3" optional_params=("${@:4}") + local -a base_command + local response process_id + + base_command=("$VERSITY_EXE" --access="$access_id" --secret="$secret_key" --region="$AWS_REGION") if [ -n "$RUN_USERS" ]; then # shellcheck disable=SC2153 IFS=' ' read -r -a iam_array <<< "$IAM_PARAMS" @@ -121,16 +127,21 @@ run_versity_app_posix() { if [ -n "$WEBSITE_DOMAIN" ]; then base_command+=(--website-domain "$WEBSITE_DOMAIN") fi + base_command+=("${optional_params[@]}") base_command+=(posix) if [ -n "$VERSIONING_DIR" ]; then base_command+=(--versioning-dir "$VERSIONING_DIR") fi base_command+=("$LOCAL_FOLDER") + log 5 "base command: ${base_command[*]}" - if ! start_versity_process "$3" "${base_command[@]}"; then - log 1 "error starting versity process" + if ! response=$(start_versity_process "$versitygw_app_index" "${base_command[@]}" 2>&1); then + log 1 "error starting versity process: $response" return 1 fi + process_id="$response" + log 5 "pid here: $process_id" + printf '%s\n' "$process_id" return 0 } @@ -177,10 +188,15 @@ run_versity_app_s3() { } run_versity_app() { + local additional_params=("$@") + local response process_id + if [[ $BACKEND == 'posix' ]]; then - if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then + if ! response=$(run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" "${additional_params[@]}" 2>&1); then + log 2 "error running POSIX versity app: $response" return 1 fi + process_id="$response" elif [[ $BACKEND == 'scoutfs' ]]; then if ! run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then return 1 @@ -196,6 +212,8 @@ run_versity_app() { log 1 "unrecognized backend type $BACKEND" return 1 fi + log 5 "and here: $process_id" + printf '%s\n' "$process_id" if [[ $IAM_TYPE != "s3" ]]; then return 0 fi