diff --git a/tests/.env.default b/tests/.env.default index b3534c64..b6767070 100644 --- a/tests/.env.default +++ b/tests/.env.default @@ -1,31 +1,79 @@ +# AWS connection AWS_PROFILE=versity AWS_ENDPOINT_URL=https://127.0.0.1:7070 +SECRETS_FILE=./tests/.secrets + +# versitygw process VERSITY_EXE=./versitygw RUN_VERSITYGW=true +#PORT=7070 +#PORT_TWO=7071 +CERT=$PWD/cert.pem +KEY=$PWD/versitygw.pem + +# backend storage BACKEND=posix LOCAL_FOLDER=/tmp/gw +VERSIONING_DIR=/tmp/versioning + +# bucket lifecycle BUCKET_ONE_NAME=versity-gwtest-bucket-one BUCKET_TWO_NAME=versity-gwtest-bucket-two RECREATE_BUCKETS=true -CERT=$PWD/cert.pem -KEY=$PWD/versitygw.pem +#DELETE_BUCKETS_AFTER_TEST=true + +# clients S3CMD_CONFIG=./tests/s3cfg.local.default -SECRETS_FILE=./tests/.secrets MC_ALIAS=versity -LOG_LEVEL=2 -GOCOVERDIR=$PWD/cover -USERS_FOLDER=$PWD/iam -#TEST_LOG_FILE=test.log -#VERSITY_LOG_FILE=versity.log + +# IAM and users IAM_TYPE=folder -DIRECT=false -#DIRECT_DISPLAY_NAME= +USERS_FOLDER=$PWD/iam +#USERS_BUCKET= USERNAME_ONE=ABCDEFG PASSWORD_ONE=HIJKLMN USERNAME_TWO=HIJKLMN PASSWORD_TWO=OPQRSTU +#AUTOGENERATE_USERS=true +#USER_AUTOGENERATION_PREFIX= +#CREATE_STATIC_USERS_IF_NONEXISTENT=false +#SKIP_USERS_TESTS=false + +# direct mode +DIRECT=false +#DIRECT_DISPLAY_NAME= +#DIRECT_AWS_USER_ID= +#DIRECT_S3_ROOT_ACCOUNT_NAME= +#DIRECT_POST_COMMAND_DELAY=0 +#DIRECT_CLOUDFRONT_TAG= + +# logs and coverage +LOG_LEVEL=2 +GOCOVERDIR=$PWD/cover +#TEST_LOG_FILE=test.log +#VERSITY_LOG_FILE=versity.log +COMMAND_LOG=command.log +TIME_LOG=time.log +#COVERAGE_LOG= +#MAX_OPENSSL_COMMAND_LOG_BYTES= +LOG_ON_SUCCESS=true +LOG_TO_STDOUT=true + +# test files and generated tools TEST_FILE_FOLDER=$PWD/versity-gwtest-files REMOVE_TEST_FILE_FOLDER=true -VERSIONING_DIR=/tmp/versioning -COMMAND_LOG=command.log -TIME_LOG=time.log \ No newline at end of file +#PYTHON_ENV_FOLDER= +#TEMPLATE_MATRIX_FILE= +#GO_COMMAND_GENERATOR_EXECUTABLE= + +# test behavior +#SKIP_ACL_TESTING=false +#MAX_FILE_DOWNLOAD_CHUNK_SIZE= +#SKIP_POLICY=false +#SKIP_BUCKET_OWNERSHIP_CONTROLS=false +#BYPASS_ENV_FILE=false +#QUICK_COMPARE_SIZE= + +# website +#WEBSITE= +#WEBSITE_ENDPOINT= diff --git a/tests/Dockerfile_test_bats b/tests/Dockerfile_test_bats index 95e0c9bb..cd8ee47f 100644 --- a/tests/Dockerfile_test_bats +++ b/tests/Dockerfile_test_bats @@ -19,7 +19,7 @@ USER tester COPY --chown=tester:tester . /home/tester USER root -RUN INSTALL_MC=true /home/tester/tests/install_test.sh +RUN /home/tester/tests/install_test.sh WORKDIR /tmp RUN update-ca-certificates && rm -rf /var/lib/apt/lists/* diff --git a/tests/README.md b/tests/README.md index 934ec67b..4c0aae3d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -63,11 +63,11 @@ The bats tests have tag headers to allow the test user to easily find tests that To preserve buckets while running tests, set `RECREATE_BUCKETS` to `false`. Two utility functions are included, if needed, to create, and delete buckets for this: `tests/setup_static.sh` and `tests/remove_static.sh`. Note that this creates a bucket with object lock enabled, and some tests may fail if the bucket being tested doesn't have object lock enabled. -### ~~S3 Backend~~ (Not Working) +### S3 Backend -Instructions are mostly the same; however, testing with the S3 backend requires two S3 accounts. Ideally, these are two real accounts, but one can also be a dummy account that versity uses internally. +The S3 backend mode allows versitygw to connect to another S3 endpoint. For example, this can connect to S3 itself in a proxy configuration, or can connect to another versitygw instance with a posix backend. This can be done for applications such as local authorization, distributed deployments, testing, etc. -To set up the latter: +To set up S3 as a proxy: 1. Create a new AWS profile with ID and key values set to dummy 20-char allcaps and 40-char alphabetical values respectively. 2. In the `.secrets` file being used, create the fields `AWS_ACCESS_KEY_ID_TWO` and `AWS_SECRET_ACCESS_KEY_TWO`. Set these values to the actual AWS ID and key. 3. Set the values for `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` the same dummy values set in the AWS profile, and set `AWS_PROFILE` to the profile you just created. @@ -174,8 +174,6 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up **TEST_LOG_FILE**: log file location for these bats tests. -**VERSITY_LOG_FILE**: log file for versity application as it is tested by bats tests. - **DIRECT**: if **true**, bypass versitygw and run directly against s3, for comparison and validity-checking purposes. This parameter disables the `AWS_ENDPOINT_URL` parameter. **DIRECT_DISPLAY_NAME**: AWS ACL main user display name if **DIRECT** is set to **true**. @@ -190,8 +188,6 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up **VERSIONING_DIR**: where to put gateway file versioning info. -**COMMAND_LOG**: where to store list of client commands, which if using will be reported during test failures. - **TIME_LOG**: optional log to show duration of individual tests **DIRECT_S3_ROOT_ACCOUNT_NAME**: for direct mode, S3 username for user with root permissions @@ -236,6 +232,10 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up **WEBSITE_ENDPOINT**: website endpoint, if using versitygw website functionality +**LOG_ON_SUCCESS**: set to `false` to delete log on test success + +**LOG_TO_STDOUT**: set to `false` to avoid logging to stdout, e.g. to make Dockerfile output more clear + ## REST Scripts REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder. To call a script, the following parameters are needed: diff --git a/tests/env.sh b/tests/env.sh index 566d8034..582fd3a3 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -22,20 +22,6 @@ if [ -n "$BASH_VERSION" ] && [ "${BASH_VERSINFO[0]}" -lt 4 ]; then exit 1 fi -base_setup() { - if ! check_env_vars; then - log 1 "error checking env vars" - return 1 - fi - if [ "$RUN_VERSITYGW" == "true" ] && [ "$UNIT_TEST" != "true" ]; then - if ! run_versity_app; then - log 1 "error running versitygw app" - return 1 - fi - fi - return 0 -} - get_log_name_from_placeholders() { local test_log_file_string="$TEST_LOG_FILE_PATTERN" timestamp test_filename filename_root @@ -54,7 +40,7 @@ get_log_name_from_placeholders() { get_log_name() { if [ -n "$TEST_LOG_FILE_PATTERN" ]; then if [ -n "$TEST_LOG_FILE" ]; then - echo "both TEST_LOG_FILE_PATTERN and TEST_LOG_FILE cannot be defined" + echo "both TEST_LOG_FILE_PATTERN and TEST_LOG_FILE cannot be defined" >&2 return 1 fi if [[ ( "$TEST_LOG_FILE_PATTERN" == *'{'* ) || ( "$TEST_LOG_FILE_PATTERN" == *'}'* ) ]]; then @@ -65,18 +51,30 @@ get_log_name() { fi } -setup_test_log_file() { +setup_test_logging() { if [ -n "$TEST_LOG_FILE" ] || [ -n "$TEST_LOG_FILE_PATTERN" ]; then if ! get_log_name; then - log 1 "error getting log file name" + echo "error getting log file name" >&2 return 1 fi if ! error=$(touch "$TEST_LOG_FILE.$TEST_ID" 2>&1); then - log 1 "error creating test log file: $error" + echo "error creating test log file: $error" >&2 return 1 fi export TEST_LOG_FILE fi + if [ -z "$LOG_ON_SUCCESS" ]; then + LOG_ON_SUCCESS="true" + elif [ "$LOG_ON_SUCCESS" != "true" ] && [ "$LOG_ON_SUCCESS" != "false" ]; then + log 1 "LOG_ON_SUCCESS must be unset (default: true), true, or false" + return 1 + fi + if [ -z "$LOG_TO_STDOUT" ]; then + LOG_TO_STDOUT="true" + elif [ "$LOG_TO_STDOUT" != "true" ] && [ "$LOG_TO_STDOUT" != "false" ]; then + log 1 "LOG_TO_STDOUT must be unset (default: true), true, or false" + return 1 + fi return 0 } @@ -205,18 +203,6 @@ check_universal_vars() { if [[ $BYPASS_ENV_FILE != "true" ]]; then source_config_file fi - if [ -n "$COMMAND_LOG" ]; then - if ! init_command_log; then - log 1 "error initializing command log" - return 1 - fi - fi - if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then - # shellcheck source=./tests/.secrets - source "$SECRETS_FILE" - else - log 3 "Warning: no secrets file found" - fi if [[ -n "$LOG_LEVEL" ]]; then if [[ $LOG_LEVEL -lt 2 ]]; then log 1 "log level must be 2 or greater" @@ -224,6 +210,27 @@ check_universal_vars() { fi export LOG_LEVEL_INT=$LOG_LEVEL fi + if [ -z "$TEST_FILE_FOLDER" ]; then + log 1 "TEST_FILE_FOLDER missing" + return 1 + fi + if [ ! -d "$TEST_FILE_FOLDER" ]; then + if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then + log 1 "error creating test folder: $error" + return 1 + fi + fi + export TEST_FILE_FOLDER + if ! init_command_log; then + log 1 "error initializing command log" + return 1 + fi + if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then + # shellcheck source=./tests/.secrets + source "$SECRETS_FILE" + else + log 3 "Warning: no secrets file found" + fi if [ "$DIRECT" == "true" ]; then if [ -z "$DIRECT_POST_COMMAND_DELAY" ]; then DIRECT_POST_COMMAND_DELAY=0 @@ -257,18 +264,6 @@ check_universal_vars() { log 1 "error checking bucket-related env vars" return 1 fi - - if [ -z "$TEST_FILE_FOLDER" ]; then - log 1 "TEST_FILE_FOLDER missing" - return 1 - fi - if [ ! -d "$TEST_FILE_FOLDER" ]; then - if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then - log 1 "error creating test folder: $error" - return 1 - fi - fi - export TEST_FILE_FOLDER return 0 } @@ -413,21 +408,11 @@ check_user_profile_and_add_if_needed() { return 0 } -delete_command_log() { - if [ -f "$COMMAND_LOG" ]; then - if ! error=$(rm "$COMMAND_LOG"); then - log 2 "error removing command log: $error" - return 1 - fi - fi - return 0 -} - init_command_log() { - if ! delete_command_log; then - log 1 "error deleting old command log" - return 1 - fi + COMMAND_LOG="$TEST_FILE_FOLDER/command-$(uuidgen).log" + log 5 "command log: $COMMAND_LOG" + + export COMMAND_LOG if ! echo "******** $(date +"%Y-%m-%d %H:%M:%S") $BATS_TEST_NAME COMMANDS ********" >> "$COMMAND_LOG"; then log 1 "fatal error: unable to write to file '$COMMAND_LOG'" return 1 @@ -437,11 +422,18 @@ init_command_log() { main_log_cleanup() { if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then - if ! error=$(cat "${TEST_LOG_FILE}.${TEST_ID}" >> "$TEST_LOG_FILE" 2>&1); then - log 2 "error appending temp log to main log: $error" + if [ "$LOG_ON_SUCCESS" == "true" ] || [ "$BATS_TEST_COMPLETED" != "1" ]; then + if ! error=$(cat "${TEST_LOG_FILE}.${TEST_ID}" >> "$TEST_LOG_FILE" 2>&1); then + echo "error appending temp log to main log '$TEST_LOG_FILE': $error" >&2 + return 1 + fi + # for docker + if [ "$BATS_TEST_COMPLETED" != "1" ]; then + echo "Log info written to '$TEST_LOG_FILE'" >&2 + fi fi if ! error=$(rm "${TEST_LOG_FILE}.${TEST_ID}" 2>&1); then - log 2 "error deleting temp log: $error" + echo "error deleting temp log: $error" >&2 return 1 fi fi @@ -449,27 +441,32 @@ main_log_cleanup() { } teardown_logs() { - local response + local response teardown_result=0 + log 5 "tearing down logs" if [[ $LOG_LEVEL -ge 4 ]] || [[ -n "$TIME_LOG" ]]; then - teardown_time_log + teardown_time_log || teardown_result=$? fi if [[ -f "${TEST_LOG_FILE}.${TEST_ID}" ]]; then echo "********************************** END TEST LOG **********************************" >> "${TEST_LOG_FILE}.${TEST_ID}" fi if [[ -f "$COMMAND_LOG" ]]; then - teardown_command_log + log 5 "tearing down command log" + teardown_command_log || teardown_result=$? fi - if [ -f "${VERSITY_LOG_FILE}.${TEST_ID}.1" ]; then - teardown_versity_log 1 + if [ -f "$VERSITY_LOG_FILE_1" ]; then + log 5 "tearing down versitygw 1 log" + teardown_versity_log "$VERSITY_LOG_FILE_1" || teardown_result=$? fi - if [ -f "${VERSITY_LOG_FILE}.${TEST_ID}.2" ]; then - teardown_versity_log 2 + if [ -f "$VERSITY_LOG_FILE_2" ]; then + log 5 "tearing down versitygw 2 log" + teardown_versity_log "$VERSITY_LOG_FILE_2" || teardown_result=$? fi - if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ] && [ "$BATS_TEST_COMPLETED" != "1" ]; then + if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ] && [ "$BATS_TEST_COMPLETED" != "1" ] && [ "$LOG_TO_STDOUT" != "false" ]; then cat "${TEST_LOG_FILE}.${TEST_ID}" fi - main_log_cleanup + main_log_cleanup || teardown_result=$? + return $teardown_result } teardown_time_log() { @@ -488,31 +485,31 @@ teardown_time_log() { } teardown_command_log() { - echo "**********************************************************************************" >> "$COMMAND_LOG" - if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then - cat "$COMMAND_LOG" >> "${TEST_LOG_FILE}.${TEST_ID}" - elif [ "$BATS_TEST_COMPLETED" != "1" ]; then - cat "$COMMAND_LOG" - fi - if ! delete_command_log; then - log 2 "error deleting command log" - return 1 - fi - return 0 + teardown_appended_log "$COMMAND_LOG" } teardown_versity_log() { - if ! check_param_count_v2 "versitygw process ID" 1 $#; then + if ! check_param_count_v2 "log name" 1 $#; then return 1 fi - echo "**********************************************************************************" >> "${VERSITY_LOG_FILE}.${TEST_ID}.$1" - if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then - cat "${VERSITY_LOG_FILE}.${TEST_ID}.$1" >> "${TEST_LOG_FILE}.${TEST_ID}" - elif [ "$BATS_TEST_COMPLETED" != "1" ]; then - cat "${VERSITY_LOG_FILE}.${TEST_ID}.$1" + teardown_appended_log "$1" +} + +teardown_appended_log() { + if ! check_param_count_v2 "log name" 1 $#; then + return 1 fi - if ! response=$(rm "${VERSITY_LOG_FILE}.${TEST_ID}.$1" 2>&1); then - log 3 "error deleting log file '${VERSITY_LOG_FILE}.${TEST_ID}.$1': $response" + local log_name="$1" response + + echo "**********************************************************************************" >> "$log_name" + if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then + cat "$log_name" >> "${TEST_LOG_FILE}.${TEST_ID}" + elif [ "$BATS_TEST_STATUS" != 0 ]; then + cat "$log_name" + fi + if ! response=$(rm "$log_name" 2>&1); then + log 3 "error deleting log file '$log_name': $response" + return 1 fi return 0 } diff --git a/tests/run_parallel.sh b/tests/run_parallel.sh index 95a80c5e..c1c7aad1 100755 --- a/tests/run_parallel.sh +++ b/tests/run_parallel.sh @@ -18,12 +18,13 @@ source ./tests/drivers/params.sh DEFAULT_MAX_PARALLEL_JOBS=4 DEFAULT_DOCKER_LOG_FOLDER="$PWD/runtime/log" +DOCKERFILE="tests/Dockerfile_test_bats" check_for_finished_processes() { - if ! check_param_count_v2 "pid array ref, suite array ref, time array ref" 3 $#; then + if ! check_param_count_v2 "pid array ref, suite array ref, time array ref, log array ref" 4 $#; then return 1 fi - local -n pids_ref="$1" suites_ref="$2" times_ref="$3" + local -n pids_ref="$1" suites_ref="$2" times_ref="$3" logs_ref="$4" local -a pid_snapshot local pid status run_end_time @@ -34,9 +35,14 @@ check_for_finished_processes() { status=$? run_end_time=$(date +%s) printf '%s\n' "'$pid' (${suites_ref[$pid]}) finished with status '$status' (duration: $((run_end_time-${times_ref[$pid]}))s)" + if [ "$status" -ne 0 ]; then + test_failure=1 + printf 'log file: %s\n' "${logs_ref[$pid]}" + fi unset "pids_ref[$pid]" unset "suites_ref[$pid]" unset "times_ref[$pid]" + unset "logs_ref[$pid]" fi done } @@ -46,35 +52,89 @@ run_tests() { return 1 fi local image_tag="$1" test_list="$2" max_parallel_jobs="$3" docker_log_folder="$4" - local test_array pids suites times timestamp test_suite end_time duration + local test_array pids suites times timestamp test_suite end_time duration log_name IFS=, read -r -a test_array <<< "$test_list" - declare -A pids suites times + declare -A pids suites times logs timestamp="$(date '+%Y%m%dT%H%M%S')" for test_suite in "${test_array[@]}"; do while [ ${#pids[@]} -ge "$max_parallel_jobs" ]; do sleep 1 - check_for_finished_processes "pids" "suites" "times" + check_for_finished_processes "pids" "suites" "times" "logs" done - docker run -v "$PWD/runtime/config:/home/tester/config" -v "$docker_log_folder:/home/tester/log" -t "$image_tag" "$test_suite" > "$docker_log_folder/${test_suite}-${timestamp}.log" & + log_name="$docker_log_folder/${test_suite}-${timestamp}.log" + docker run -v "$PWD/runtime/config:/home/tester/config" -v "$docker_log_folder:/home/tester/log" -t "$image_tag" "$test_suite" > "$log_name" & pid=$! printf '%s\n' "'$test_suite' started (pid: '$pid')" pids[$pid]=$pid # shellcheck disable=SC2034 suites[$pid]=$test_suite # shellcheck disable=SC2034 + logs[$pid]=$log_name + # shellcheck disable=SC2034 times[$pid]=$(date +%s) done while [ ${#pids[@]} -gt 0 ]; do - check_for_finished_processes "pids" "suites" "times" + check_for_finished_processes "pids" "suites" "times" "logs" sleep 1 done } -if ! check_param_count_ge_le "docker image tag, test suites (separated by comma), max parallel jobs at once (default '$DEFAULT_MAX_PARALLEL_JOBS', log folder (default '$DEFAULT_DOCKER_LOG_FOLDER')" 2 4 $#; then +usage() { + printf '%s\n' "Usage: $0 [max parallel jobs] [log folder] [--rebuild|--build-if-missing|--use-tag]" + printf '%s\n' " --rebuild Rebuild the Docker image before testing." + printf '%s\n' " --build-if-missing Build the Docker image only if the tag does not exist." + printf '%s\n' " --use-tag Use the provided tag without building (default)." +} + +build_container() { + if ! check_param_count_v2 "image tag" 1 $#; then + return 1 + fi + local image_tag="$1" + + docker build -t "$image_tag" -f "$DOCKERFILE" . +} + +prepare_container() { + if ! check_param_count_v2 "image tag, build mode" 2 $#; then + return 1 + fi + local image_tag="$1" build_mode="$2" + + case "$build_mode" in + rebuild) + build_container "$image_tag" + ;; + build-if-missing) + if ! docker image inspect "$image_tag" >/dev/null 2>&1; then + build_container "$image_tag" + fi + ;; + use-tag) + if ! docker image inspect "$image_tag" >/dev/null 2>&1; then + echo "image tagged '$image_tag' doesn't exist" + return 1 + fi + ;; + *) + echo "unknown build mode '$build_mode'" + usage + return 1 + ;; + esac +} + +if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then + usage + exit 0 +fi + +if [ $# -lt 2 ]; then + usage exit 1 fi @@ -82,23 +142,48 @@ start_time=$(date +%s) image_tag="$1" test_list="$2" -if [ "$3" == "" ]; then - max_parallel_jobs="$DEFAULT_MAX_PARALLEL_JOBS" -else - if [ "$3" -le 0 ]; then - echo "max parallel jobs must be at least 1" - exit 1 - fi - max_parallel_jobs="$3" -fi -if [ "$4" == "" ]; then - docker_log_folder="$DEFAULT_DOCKER_LOG_FOLDER" -else - docker_log_folder="$4" -fi +max_parallel_jobs="$DEFAULT_MAX_PARALLEL_JOBS" +docker_log_folder="$DEFAULT_DOCKER_LOG_FOLDER" +build_mode="use-tag" +docker_log_folder_set=false +shift 2 -if ! docker image inspect "$image_tag" >/dev/null 2>&1; then - echo "image tagged '$image_tag' doesn't exist" +while [ $# -gt 0 ]; do + case "$1" in + --rebuild) + build_mode="rebuild" + ;; + --build-if-missing) + build_mode="build-if-missing" + ;; + --use-tag) + build_mode="use-tag" + ;; + -h|--help) + usage + exit 0 + ;; + *) + if [[ "$1" =~ ^[0-9]+$ ]]; then + if [ "$1" -le 0 ]; then + echo "max parallel jobs must be at least 1" + exit 1 + fi + max_parallel_jobs="$1" + elif [ "$docker_log_folder_set" == "false" ]; then + docker_log_folder="$1" + docker_log_folder_set=true + else + echo "unknown option or extra parameter '$1'" + usage + exit 1 + fi + ;; + esac + shift +done + +if ! prepare_container "$image_tag" "$build_mode"; then exit 1 fi @@ -107,8 +192,11 @@ if ! mkdir -p "$docker_log_folder"; then exit 1 fi +test_failure=0 run_tests "$image_tag" "$test_list" "$max_parallel_jobs" "$docker_log_folder" end_time=$(date +%s) duration=$((end_time-start_time)) printf '%s\n' "duration: ${duration}s" + +exit "$test_failure" diff --git a/tests/setup_common.sh b/tests/setup_common.sh index 3558fd6e..6e2d1626 100644 --- a/tests/setup_common.sh +++ b/tests/setup_common.sh @@ -26,8 +26,8 @@ setup_env() { source_config_file - if ! setup_test_log_file; then - log 1 "error creating test log file" + if ! setup_test_logging; then + echo "error setting up test logging" >&2 return 1 fi @@ -114,7 +114,7 @@ setup_clients() { } teardown_common() { - local response proc_status_one proc_status_two + local response proc_status_one proc_status_two teardown_status=0 response=$(check_versity_process_status) read -r proc_status_one proc_status_two <<< "$response" @@ -124,25 +124,34 @@ teardown_common() { if [ "$proc_status_one" == "none" ] || [[ ( "$proc_status_one" == "running" ) && (( "$proc_status_two" == "none") || ( "$proc_status_two" == "running")) ]]; then if ! bucket_and_user_cleanup; then log 3 "bucket and user cleanup not properly done" + teardown_status=1 fi fi if [ "$proc_status_one" == "running" ]; then if ! stop_versity_process "$VERSITYGW_PID_1"; then log 3 "unable to properly stop versitygw process" + teardown_status=1 fi fi if [ "$proc_status_two" == "running" ]; then if ! stop_versity_process "$VERSITYGW_PID_2"; then log 3 "unable to properly stop second versity process" + teardown_status=1 fi fi - if ! remove_test_file_folder_if_desired; then - log 3 "test file folder cleanup error" + # for docker failures + if [ "${BATS_TEST_STATUS:-0}" -ne 0 ] && [ -n "$TEST_LOG_FILE" ]; then + log 2 "test log file location: '${TEST_LOG_FILE}.${TEST_ID}'" fi if ! teardown_logs; then log 3 "log file teardown error" + teardown_status=1 fi - return 0 + if ! remove_test_file_folder_if_desired; then + log 3 "test file folder cleanup error" + teardown_status=1 + fi + return $teardown_status } static_user_v1_cleanup() { diff --git a/tests/setup_unit.sh b/tests/setup_unit.sh index d6cd9007..1fd2983c 100644 --- a/tests/setup_unit.sh +++ b/tests/setup_unit.sh @@ -14,15 +14,22 @@ # specific language governing permissions and limitations # under the License. -source ./tests/env.sh +source ./tests/setup_common.sh export UNIT_TEST=true setup() { - base_setup - setup_test_log_file + if ! setup_env; then + echo "error with env setup" >&2 + return 1 + fi + return 0 } teardown() { - teardown_logs + if ! teardown_logs; then + echo "log teardown errors" >&2 + return 1 + fi + return 0 } \ No newline at end of file diff --git a/tests/test_command.sh b/tests/test_command.sh index 56d44cb5..5efce145 100755 --- a/tests/test_command.sh +++ b/tests/test_command.sh @@ -41,3 +41,34 @@ source ./tests/setup_unit.sh run check_for_and_or_build_go_executable assert_success } + +@test "Run parallel test, print log on failure" { + local fake_bin log_folder matching_logs + + fake_bin="$TEST_FILE_FOLDER/bin" + log_folder="$TEST_FILE_FOLDER/run_parallel_logs" + mkdir -p "$fake_bin" "$log_folder" + + cat > "$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash + +if [ "$1" == "image" ] && [ "$2" == "inspect" ]; then + exit 0 +fi +if [ "$1" == "run" ]; then + printf '%s\n' "fake test failure" + exit 7 +fi +printf 'unexpected docker args: %s\n' "$*" >&2 +exit 1 +EOF + chmod +x "$fake_bin/docker" + + PATH="$fake_bin:$PATH" run ./tests/run_parallel.sh "fake-image" "failing-suite" 1 "$log_folder" --use-tag + + assert_failure + assert_output -p "finished with status '7'" + assert_output -p "log file: $log_folder/failing-suite-" + matching_logs=("$log_folder"/failing-suite-*.log) + assert [ -f "${matching_logs[0]}" ] +} diff --git a/tests/test_env.sh b/tests/test_env.sh new file mode 100755 index 00000000..bc0a4107 --- /dev/null +++ b/tests/test_env.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bats + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/logger.sh +source ./tests/drivers/params.sh +source ./tests/env.sh +source ./tests/setup_unit.sh + +# tags: unit +@test "teardown time log - elapsed time written" { + local duration log_line + + TIME_LOG="$TEST_FILE_FOLDER/time-$(uuidgen).log" + BATS_TEST_NAME="time log test" + START_TIME=$(date +%s) + + run teardown_time_log + + assert_success + assert [ -f "$TIME_LOG" ] + log_line=$(<"$TIME_LOG") + duration=${log_line#"time log test: "} + duration=${duration%"s"} + assert [ "$log_line" != "$duration" ] + assert [ "$duration" -ge 0 ] + assert [ "$duration" -le 2 ] +} + +# tags: unit +@test "teardown time log - succeeds without TIME_LOG" { + unset TIME_LOG + BATS_TEST_NAME="time log unset test" + START_TIME=$(date +%s) + + run teardown_time_log + + assert_success + assert_output "" +} + +# tags: unit +@test "teardown appended log - appends to existing test log" { + local test_log appended_log contents + + TEST_ID="appended-log-$(uuidgen)" + TEST_LOG_FILE="$TEST_FILE_FOLDER/test-$(uuidgen).log" + appended_log="$TEST_FILE_FOLDER/appended-$(uuidgen).log" + test_log="${TEST_LOG_FILE}.${TEST_ID}" + printf '%s\n' "existing test log" > "$test_log" + printf '%s\n' "appended log entry" > "$appended_log" + + run teardown_appended_log "$appended_log" + + assert_success + assert_output "" + assert [ ! -f "$appended_log" ] + contents=$(<"$test_log") + [[ "$contents" == *"existing test log"* ]] + [[ "$contents" == *"appended log entry"* ]] + [[ "$contents" == *"**********************************************************************************"* ]] +} + +# tags: unit +@test "teardown appended log - prints when test log is missing and test fails" { + local appended_log + + TEST_ID="appended-log-missing-$(uuidgen)" + TEST_LOG_FILE="$TEST_FILE_FOLDER/test-$(uuidgen).log" + BATS_TEST_COMPLETED=0 + appended_log="$TEST_FILE_FOLDER/appended-$(uuidgen).log" + printf '%s\n' "failed appended log entry" > "$appended_log" + + run teardown_appended_log "$appended_log" + + assert_success + assert_output -p "failed appended log entry" + assert_output -p "**********************************************************************************" + assert [ ! -f "$appended_log" ] + assert [ ! -f "${TEST_LOG_FILE}.${TEST_ID}" ] +} + +# tags: unit +@test "teardown appended log - deletes only when test log is missing and test passes" { + local appended_log + + TEST_ID="appended-log-complete-$(uuidgen)" + TEST_LOG_FILE="$TEST_FILE_FOLDER/test-$(uuidgen).log" + BATS_TEST_COMPLETED=1 + appended_log="$TEST_FILE_FOLDER/appended-$(uuidgen).log" + printf '%s\n' "completed appended log entry" > "$appended_log" + + run teardown_appended_log "$appended_log" + + assert_success + assert_output "" + assert [ ! -f "$appended_log" ] + assert [ ! -f "${TEST_LOG_FILE}.${TEST_ID}" ] +} + +# tags: unit +@test "teardown appended log - fails when delete fails" { + local fake_bin test_log appended_log contents + + fake_bin="$TEST_FILE_FOLDER/bin-$(uuidgen)" + TEST_ID="appended-log-delete-fail-$(uuidgen)" + TEST_LOG_FILE="$TEST_FILE_FOLDER/test-$(uuidgen).log" + test_log="${TEST_LOG_FILE}.${TEST_ID}" + appended_log="$TEST_FILE_FOLDER/appended-$(uuidgen).log" + mkdir -p "$fake_bin" + printf '%s\n' "existing test log" > "$test_log" + printf '%s\n' "appended log entry" > "$appended_log" + cat > "$fake_bin/rm" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "fake rm failure" >&2 +exit 1 +EOF + chmod +x "$fake_bin/rm" + + PATH="$fake_bin:$PATH" run teardown_appended_log "$appended_log" + + assert_failure 1 + assert_output "" + assert [ -f "$appended_log" ] + contents=$(<"$test_log") + [[ "$contents" == *"error deleting log file"* ]] + [[ "$contents" == *"fake rm failure"* ]] +} + +# tags: unit +@test "teardown command log - uses COMMAND_LOG" { + local test_log contents + + TEST_ID="command-log-$(uuidgen)" + TEST_LOG_FILE="$TEST_FILE_FOLDER/test-$(uuidgen).log" + COMMAND_LOG="$TEST_FILE_FOLDER/command-$(uuidgen).log" + test_log="${TEST_LOG_FILE}.${TEST_ID}" + printf '%s\n' "existing test log" > "$test_log" + printf '%s\n' "command log entry" > "$COMMAND_LOG" + + run teardown_command_log + + assert_success + assert_output "" + assert [ ! -f "$COMMAND_LOG" ] + contents=$(<"$test_log") + [[ "$contents" == *"command log entry"* ]] +} + +# tags: unit +@test "main log cleanup - fails when appending temp log fails" { + local main_log temp_log + + TEST_ID="main-log-append-fail-$(uuidgen)" + TEST_LOG_FILE="$TEST_FILE_FOLDER/main-log-$(uuidgen)" + BATS_TEST_COMPLETED=0 + main_log="$TEST_LOG_FILE" + temp_log="${TEST_LOG_FILE}.${TEST_ID}" + # induce failure by making main log a folder rather than a file + mkdir -p "$main_log" + printf '%s\n' "temp test log entry" > "$temp_log" + + run main_log_cleanup + + TEST_LOG_FILE= + + assert_failure 1 + assert_output -p "error appending temp log to main log" + assert [ -d "$main_log" ] + assert [ -f "$temp_log" ] +} + +# tags: unit +@test "main log cleanup - skips append when LOG_ON_SUCCESS is false" { + local main_log temp_log contents + + TEST_ID="main-log-success-skip-$(uuidgen)" + TEST_LOG_FILE="$TEST_FILE_FOLDER/main-log-$(uuidgen).log" + BATS_TEST_COMPLETED=1 + LOG_ON_SUCCESS=false + main_log="$TEST_LOG_FILE" + temp_log="${TEST_LOG_FILE}.${TEST_ID}" + printf '%s\n' "existing main log" > "$main_log" + printf '%s\n' "successful temp log entry" > "$temp_log" + + run main_log_cleanup + + assert_success + assert_output "" + assert [ ! -f "$temp_log" ] + contents=$(<"$main_log") + [[ "$contents" == *"existing main log"* ]] + [[ "$contents" != *"successful temp log entry"* ]] +} + +# tags: unit +@test "teardown versity log - uses provided log name" { + local test_log versity_log contents + + TEST_ID="versity-log-$(uuidgen)" + TEST_LOG_FILE="$TEST_FILE_FOLDER/test-$(uuidgen).log" + test_log="${TEST_LOG_FILE}.${TEST_ID}" + versity_log="$TEST_FILE_FOLDER/versity-$(uuidgen).log" + printf '%s\n' "existing test log" > "$test_log" + printf '%s\n' "versity log entry" > "$versity_log" + + run teardown_versity_log "$versity_log" + + assert_success + assert_output "" + assert [ ! -f "$versity_log" ] + contents=$(<"$test_log") + [[ "$contents" == *"versity log entry"* ]] +} diff --git a/tests/test_report.sh b/tests/test_report.sh index 30974123..1d695d5c 100755 --- a/tests/test_report.sh +++ b/tests/test_report.sh @@ -79,11 +79,11 @@ source ./tests/setup_unit.sh expected_output=("GET MAIN " "PUT OBJECT prefix") for ((i=0; i<${#tests[@]}; i++)); do - if file_name=$(get_file_name_with_prefix "openssl" 2>&1); then + if ! file_name=$(get_file_name_with_prefix "openssl" 2>&1); then return 1 fi - echo "${tests[$i]}" > "$file_name" - run get_openssl_method_route_queries "$file_name" + echo "${tests[$i]}" > "${TEST_FILE_FOLDER}/${file_name}" + run get_openssl_method_route_queries "${TEST_FILE_FOLDER}/${file_name}" assert_output "${expected_output[$i]}" done } @@ -105,11 +105,11 @@ source ./tests/setup_unit.sh for ((i=0; i<${#test_clients[@]}; i++)); do echo "test $i" if [ "${test_clients[$i]}" == "OPENSSL" ]; then - if file_name=$(get_file_name_with_prefix "openssl" 2>&1); then + if ! file_name=$(get_file_name_with_prefix "openssl" 2>&1); then return 1 fi - echo "${test_data[$i]}" > "$file_name" - data_param=$file_name + echo "${test_data[$i]}" > "${TEST_FILE_FOLDER}/${file_name}" + data_param="${TEST_FILE_FOLDER}/${file_name}" else data_param=${test_data[$i]} fi diff --git a/tests/versity.sh b/tests/versity.sh index a0dd66a4..e394ae3c 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -20,27 +20,32 @@ start_versity_process() { if ! check_param_count_gt "versity app index, command array" 2 $#; then exit 1 fi - local response process_id + local versity_app_index="$1" command_array=("${@:2}") + local log_file_name response process_id - if ! response=$(build_run_and_log_command "$1" "${@:2}" 2>&1); then + log_file_name="${TEST_FILE_FOLDER}/versity-${TEST_ID}-${versity_app_index}-$(uuidgen)" + printf -v "VERSITY_LOG_FILE_${versity_app_index}" '%s' "$log_file_name" + export VERSITY_LOG_FILE_"${versity_app_index}" + + if ! response=$(build_run_and_log_command "$versity_app_index" "$log_file_name" "${command_array[@]}" 2>&1); then log 2 "error building, logging, and/or running 'versitygw' executable: $response" return 1 fi process_id="$response" - printf -v "VERSITYGW_PID_$1" '%s' "$process_id" - log 4 "versitygw PID for $1: $process_id" - export VERSITYGW_PID_"$1" + printf -v "VERSITYGW_PID_${versity_app_index}" '%s' "$process_id" + log 4 "versitygw PID for ${versity_app_index}: $process_id" + export VERSITYGW_PID_"${versity_app_index}" return 0 } build_run_and_log_command() { - if ! check_param_count_gt "versitygw process number (1 or 2), command array" 2 $#; then + if ! check_param_count_gt "versitygw process number (1 or 2), log file, command array" 3 $#; then return 1 fi - local process_number="$1" command_array=("${@:2}") - local response full_command versitygw_log_file_name="" pid check_result + local process_number="$1" log_file="$2" command_array=("${@:3}") + local response full_command pid check_result IFS=' ' read -r -a full_command <<< "${command_array[@]}" log 5 "versity command: ${full_command[*]}" @@ -52,16 +57,11 @@ build_run_and_log_command() { # shellcheck disable=SC2154 echo "$response" >> "$COMMAND_LOG" fi - if [ -n "$VERSITY_LOG_FILE" ]; then - versitygw_log_file_name="$VERSITY_LOG_FILE.$TEST_ID".$process_number - printf '****************************** VERSITYGW %s LOG \***********************************\n' "$1" >> "$versitygw_log_file_name" - "${full_command[@]}" >> "$versitygw_log_file_name" 2>&1 & - else - "${full_command[@]}" >/dev/null 2>&1 & - fi + printf '****************************** VERSITYGW %s LOG \***********************************\n' "$process_number" >> "$log_file" + "${full_command[@]}" >> "$log_file" 2>&1 & pid="$!" - if ! verify_process_started "$process_number" "$pid" "$versitygw_log_file_name"; then + if ! verify_process_started "$process_number" "$pid" "$log_file"; then return 1 fi