From e9cc750c5f695fea1df084d1b686ebbf330602ca Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Fri, 21 Aug 2026 18:12:08 -0300 Subject: [PATCH] test: safety commit --- .github/workflows/system.yml | 12 ++- tests/setup.sh | 61 +----------- ...p_env_and_versitygw.sh => setup_common.sh} | 11 +-- tests/setup_static.sh | 2 +- tests/test_rest_website.sh | 67 +++++-------- tests/versity.sh | 98 ++++++++++++------- 6 files changed, 104 insertions(+), 147 deletions(-) rename tests/{setup_env_and_versitygw.sh => setup_common.sh} (94%) diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index cbd71e79..ce7a091d 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -76,6 +76,10 @@ jobs: AWS_REGION: ${{ matrix.AWS_REGION }} COVERAGE_LOG: /home/tester/test-files/coverage.log TEMPLATE_MATRIX_FILE: ${{ github.workspace }}/tests/templates/matrix.yaml + QUICK_COMPARE_SIZE: 1048576 + GENERATE_COMMAND_EXECUTABLE: /home/tester/test-files/generateCommand + WEBSITE_DOMAIN: localhost + WEBSITE: :7071 VERSITY_LOG_FILE: /home/tester/test-files/versity.log run: | cat > /home/tester/test-files/.env <&1); then + if ! setup_versitygw; then log 1 "error starting versitygw app: $response" return 1 fi - export VERSITYGW_PID_1="$response" if ! setup_clients; then log 1 "error setting up clients" return 1 fi -# TEST_ID=$(date +"%Y%m%d-%H%M%S").$(basename "$BATS_TEST_FILENAME").$BATS_TEST_NAME -# export TEST_ID -# -# source_config_file -# -# if ! setup_test_log_file; then -# log 1 "error creating test log file" -# return 1 -# fi -# -# if [ -n "$TEST_LOG_FILE" ]; then -# printf "\n%s\n\n" "**** $TEST_ID ****" >> "${TEST_LOG_FILE}.${TEST_ID}" -# fi -# -# if ! base_setup; then -# log 2 "error checking env vars or running versitygw" -# return 1 -# fi -# -# if [ "$RUN_USERS" == "true" ] && [ "$SKIP_USERS_TESTS" != "true" ]; then -# if ! static_user_v1_cleanup; then -# log 2 "error cleaning up v1 static users" -# return 1 -# fi -# if [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ] && [ "$AUTOGENERATE_USERS" == "false" ]; then -# if ! static_user_versitygw_setup; then -# log 2 "error setting up static versitygw users" -# return 1 -# fi -# fi -# fi -# -# log 4 "Running test $BATS_TEST_NAME" -# if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then -# START_TIME=$(date +%s) -# export START_TIME -# fi -# -# if [[ $RUN_S3CMD == true ]]; then -# S3CMD_OPTS=() -# S3CMD_OPTS+=(-c "$S3CMD_CONFIG") -# S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID") -# S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY") -# S3CMD_OPTS+=(--region="$AWS_REGION") -# export S3CMD_CONFIG S3CMD_OPTS -# fi -# -# if [[ $RUN_MC == true ]] && ! check_add_mc_alias; then -# log 1 "error checking for or adding mc alias" -# return 1 -# fi -# -# export AWS_PROFILE -# -# log 4 "********** END SETUP **********" -# return 0 } # bats teardown function diff --git a/tests/setup_env_and_versitygw.sh b/tests/setup_common.sh similarity index 94% rename from tests/setup_env_and_versitygw.sh rename to tests/setup_common.sh index c75d7087..ba0a125a 100644 --- a/tests/setup_env_and_versitygw.sh +++ b/tests/setup_common.sh @@ -58,11 +58,10 @@ setup_versitygw() { local response pid if [ "$RUN_VERSITYGW" == "true" ] && [ "$UNIT_TEST" != "true" ]; then - if ! response=$(run_versity_app "${params[@]}" 2>&1); then - log 1 "error running versitygw app: $response" + if ! run_versity_app "${params[@]}"; then + log 1 "error running versitygw app" return 1 fi - pid="$response" fi if [ "$RUN_USERS" == "true" ] && [ "$SKIP_USERS_TESTS" != "true" ]; then @@ -79,7 +78,6 @@ setup_versitygw() { fi log 4 "********** END VERSITYGW SETUP **********" - printf '%s\n' "$pid" return 0 } @@ -90,11 +88,10 @@ setup_env_and_versitygw() { log 1 "error setting up env" return 1 fi - if ! response=$(setup_versitygw "$@" 2>&1); then - log 2 "error setting up versitygw: $response" + if ! setup_versitygw "$@"; then + log 2 "error setting up versitygw" return 1 fi - export VERSITYGW_PID_1="$response" return 0 } diff --git a/tests/setup_static.sh b/tests/setup_static.sh index f8ce8a0f..cdaa1c73 100755 --- a/tests/setup_static.sh +++ b/tests/setup_static.sh @@ -15,7 +15,7 @@ # under the License. source ./tests/logger.sh -source ./tests/setup_env_and_versitygw.sh +source ./tests/setup_common.sh source ./tests/util/util_object.sh source ./tests/commands/create_bucket.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh diff --git a/tests/test_rest_website.sh b/tests/test_rest_website.sh index ebaabbb0..c309f9fb 100755 --- a/tests/test_rest_website.sh +++ b/tests/test_rest_website.sh @@ -23,7 +23,7 @@ source ./tests/drivers/get_bucket_website/get_bucket_website_rest.sh source ./tests/drivers/put_bucket_website/put_bucket_website_rest.sh source ./tests/drivers/cloudfront.sh source ./tests/drivers/string.sh -source ./tests/setup_env_and_versitygw.sh +source ./tests/setup_common.sh setup() { if ! setup_env; then @@ -32,6 +32,18 @@ setup() { fi } +setup_versitygw_for_website() { + optional_params=("$@") + + run setup_versitygw "${optional_params[@]}" + assert_success + read -r process_id process_id_two <<< "$output" + export VERSITYGW_PID_1="$process_id" + if [ -n "$process_id_two" ]; then + export VERSITYGW_PID_2="$process_id_two" + fi +} + teardown() { teardown_common } @@ -39,10 +51,7 @@ teardown() { @test "PutBucketWebsite - empty payload" { local bucket_name - run setup_versitygw - assert_success - process_id="$output" - export VERSITYGW_PID_1="$process_id" + setup_versitygw_for_website run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success @@ -58,10 +67,7 @@ teardown() { fi local bucket_name - run setup_versitygw - assert_success - process_id="$output" - export VERSITYGW_PID_1="$process_id" + setup_versitygw_for_website run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success @@ -78,32 +84,17 @@ teardown() { fi distribution_created=false - run setup_versitygw - assert_success - process_id="$output" - export VERSITYGW_PID_1="$process_id" + setup_versitygw_for_website - local bucket_name policy_file distribution_domain http_domain + local bucket_name distribution_domain http_domain run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success bucket_name="$output" - run create_website_with_random_string "$bucket_name" - assert_success - random_string="$output" - - if [ "$DIRECT" == "true" ]; then - run put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false" - assert_success - fi - - run setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" - assert_success - policy_file="$output" - - run put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file" + run create_website_with_random_string_and_add_permissions "$bucket_name" assert_success + read -r random_string <<< "$output" if [ "$DIRECT" == "true" ]; then run create_cloudfront_distribution "$bucket_name" "index.html" "${bucket_name}.s3-website.us-east-1.amazonaws.com" @@ -122,12 +113,9 @@ teardown() { } @test "PutBucketWebsite - IndexDocument suffix" { - local bucket_name policy_file random_string + local bucket_name random_string - run setup_versitygw - assert_success - process_id="$output" - export VERSITYGW_PID_1="$process_id" + setup_versitygw_for_website run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success @@ -145,10 +133,7 @@ teardown() { @test "REST - GetBucketWebsite - IndexDocument Suffix, DeleteBucketWebsite" { local bucket_name test_file random_string - run setup_versitygw - assert_success - process_id="$output" - export VERSITYGW_PID_1="$process_id" + setup_versitygw_for_website run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success @@ -170,11 +155,9 @@ teardown() { } @test "REST - GetBucketWebsite - no HTTPS" { - run setup_versitygw "--website-no-tls" - assert_success - process_id="$output" - log 5 "process ID: $process_id" - export VERSITYGW_PID_1="$process_id" + local bucket_name random_string + + setup_versitygw_for_website "--website-no-tls" run setup_bucket_v3 "$BUCKET_ONE_NAME" assert_success diff --git a/tests/versity.sh b/tests/versity.sh index 36b1b02a..9b229e1a 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -32,7 +32,6 @@ start_versity_process() { log 4 "versitygw PID for $1: $process_id" export VERSITYGW_PID_"$1" - printf '%s\n' "$process_id" return 0 } @@ -62,7 +61,7 @@ build_run_and_log_command() { fi pid="$!" - if ! verify_process_started "$pid" "$versitygw_log_file_name"; then + if ! verify_process_started "$process_number" "$pid" "$versitygw_log_file_name"; then return 1 fi @@ -70,20 +69,42 @@ build_run_and_log_command() { return 0 } +get_app_two_endpoint_url() { + local endpoint="" + + if [ -n "$CERT" ]; then + endpoint="https://localhost" + else + endpoint="http://localhost" + fi + if [ -n "$PORT_TWO" ]; then + endpoint+="$PORT_TWO" + else + endpoint+=":7071" + fi + printf '%s\n' "$endpoint" + return 0 +} + verify_process_started() { - if ! check_param_count_ge_le "pid, log file (if any)" 1 2 $#; then + if ! check_param_count_ge_le "app ID, pid, log file (if any)" 2 3 $#; then return 1 fi - local check_result process_running="false" proc_state + local app_id="$1" pid="$2" log_file="$3" + local check_result process_running="false" proc_state params for ((check_num=1; check_num<=3; check_num++)); do sleep 1 - if [ "$process_running" == "false" ] && check_result=$(kill -0 "$1" 2>&1); then + if [ "$process_running" == "false" ] && check_result=$(kill -0 "$pid" 2>&1); then process_running="true" fi if [ "$process_running" == "true" ]; then - proc_state=$(ps -p "$1" -o state= 2>/dev/null | tr -d ' ') - if [ "$proc_state" != "Z" ] && list_buckets_rest "" "" >/dev/null; then + proc_state=$(ps -p "$pid" -o state= 2>/dev/null | tr -d ' ') + if [ "$app_id" == 2 ]; then + endpoint_url="$(get_app_two_endpoint_url)" + params="AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_TWO AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_TWO AWS_ENDPOINT_URL=$endpoint_url" + fi + if [ "$proc_state" != "Z" ] && list_buckets_rest "$params" "" >/dev/null; then return 0 fi fi @@ -96,8 +117,8 @@ verify_process_started() { else log 1 "process running in state '$proc_state', but not communicating properly" fi - if [[ -n "$2" ]]; then - log 1 "log data: '$(cat "$2")'" + if [[ -n "$log_file" ]]; then + log 1 "log data: '$(cat "$log_file")'" fi return 1 } @@ -119,7 +140,13 @@ run_versity_app_posix() { if [ -n "$CERT" ] && [ -n "$KEY" ]; then base_command+=(--cert "$CERT" --key "$KEY") fi - if [ -n "$PORT" ]; then + if [ "$BACKEND" == "s3" ]; then + if [ -n "$PORT_TWO" ]; then + base_command+=(--port ":$PORT_TWO") + else + base_command+=(--port ":7071") + fi + elif [ -n "$PORT" ]; then base_command+=(--port ":$PORT") fi if [ -n "$WEBSITE" ]; then @@ -136,13 +163,10 @@ run_versity_app_posix() { base_command+=("$LOCAL_FOLDER") log 5 "base command: ${base_command[*]}" - if ! response=$(start_versity_process "$versitygw_app_index" "${base_command[@]}" 2>&1); then - log 1 "error starting versity process: $response" + if ! start_versity_process "$versitygw_app_index" "${base_command[@]}"; then + log 1 "error starting versity process" return 1 fi - process_id="$response" - log 5 "pid here: $process_id" - printf '%s\n' "$process_id" return 0 } @@ -150,7 +174,11 @@ run_versity_app_scoutfs() { if ! check_param_count "run_versity_app_scoutfs" "access ID, secret key, versityid app index" 3 $#; then return 1 fi - base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION" --iam-dir="$USERS_FOLDER") + local access_id="$1" secret_key="$2" app_index="$3" + local -a base_command + local response process_id + + base_command=("$VERSITY_EXE" --access="$access_id" --secret="$secret_key" --region="$AWS_REGION" --iam-dir="$USERS_FOLDER") if [ -n "$CERT" ] && [ -n "$KEY" ]; then base_command+=(--cert "$CERT" --key "$KEY") fi @@ -159,29 +187,29 @@ run_versity_app_scoutfs() { fi base_command+=(scoutfs "$LOCAL_FOLDER") - if ! start_versity_process "$3" "${base_command[@]}"; then - log 1 "error starting versity process" + if ! start_versity_process "$app_index" "${base_command[@]}"; then + log 1 "error starting versity process: $response" return 1 fi return 0 } run_versity_app_s3() { - if ! check_param_count "run_versity_app_s3" "versityid app index" 1 $#; then + if ! check_param_count "run_versity_app_s3" "versityid app index, server key ID, server access key, dest key ID, dest access key" 5 $#; then return 1 fi - base_command=("$VERSITY_EXE" --access="$AWS_ACCESS_KEY_ID" --secret="$AWS_SECRET_ACCESS_KEY" --region="$AWS_REGION") + local app_index="$1" server_key_id="$2" server_access_key="$3" dest_key_id="$4" dest_access_key="$5" + + base_command=("$VERSITY_EXE" --access="$server_key_id" --secret="$server_access_key" --region="$AWS_REGION") if [ -n "$CERT" ] && [ -n "$KEY" ]; then base_command+=(--cert "$CERT" --key "$KEY") fi - if [ -n "$PORT_TWO" ]; then - base_command+=(--port ":$PORT_TWO") - else - base_command+=(--port ":7071") + if [ -n "$PORT" ]; then + base_command+=(--port ":$PORT") fi - base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com) + base_command+=(s3 --access="$dest_key_id" --secret="$dest_access_key" --region="$AWS_REGION" --endpoint="$AWS_ENDPOINT_URL") - if ! start_versity_process "$1" "${base_command[@]}"; then + if ! start_versity_process "$app_index" "${base_command[@]}"; then log 2 "error starting versity process" return 1 fi @@ -193,28 +221,30 @@ run_versity_app() { local response process_id if [[ $BACKEND == 'posix' ]]; then - if ! response=$(run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" "${additional_params[@]}" 2>&1); then + if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" "${additional_params[@]}"; then log 2 "error running POSIX versity app: $response" return 1 fi - process_id="$response" elif [[ $BACKEND == 'scoutfs' ]]; then if ! run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then + log 2 "error running scoutfs versity app: $response" return 1 fi elif [[ $BACKEND == 's3' ]]; then - if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then + # start in reverse order to check with ListBuckets for both + if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID_TWO" "$AWS_SECRET_ACCESS_KEY_TWO" "2"; then + log 2 "error running versitygw with posix backend in s3 config" return 1 fi - if ! run_versity_app_s3 "2"; then + if ! run_versity_app_s3 "1" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_ACCESS_KEY_ID_TWO" "$AWS_SECRET_ACCESS_KEY_TWO"; then + log 2 "error running versitygw with s3 backend in s3 config" return 1 fi else log 1 "unrecognized backend type $BACKEND" return 1 fi - log 5 "and here: $process_id" - printf '%s\n' "$process_id" + printf '%s %s\n' "$VERSITYGW_PID_1" "$VERSITYGW_PID_2" if [[ $IAM_TYPE != "s3" ]]; then return 0 fi @@ -249,13 +279,13 @@ check_versity_process_status() { status_one="none" status_two="none" if [ "$RUN_VERSITYGW" == "true" ]; then - if [[ -n "$VERSITYGW_PID_1" ]] && verify_process_started "$VERSITYGW_PID_1" >/dev/null; then + if [[ -n "$VERSITYGW_PID_1" ]] && verify_process_started "1" "$VERSITYGW_PID_1" >/dev/null; then status_one="running" else status_one="failed" fi if [ "$BACKEND" == "s3" ]; then - if [ -n "$VERSITYGW_PID_2" ] && verify_process_started "$VERSITYGW_PID_2" >/dev/null; then + if [ -n "$VERSITYGW_PID_2" ] && verify_process_started "2" "$VERSITYGW_PID_2" >/dev/null; then status_two="running" else status_two="failed"