diff --git a/tests/commands/command.sh b/tests/commands/command.sh index 686cf345..6192774a 100644 --- a/tests/commands/command.sh +++ b/tests/commands/command.sh @@ -21,17 +21,22 @@ send_command() { if ! check_param_count_gt "command data" 1 $#; then return 1 fi + + local response if [ -n "$COMMAND_LOG" ] || [ -n "$COVERAGE_LOG" ]; then args=(AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" "$@") - if ! mask_arg_array "${args[@]}"; then + if ! response=$(mask_arg_array "${args[@]}" 2>&1); then + echo "error masking args: $response" return 1 fi + masked_args="$response" + if [ -n "$COMMAND_LOG" ]; then # shellcheck disable=SC2154 - echo "${masked_args[*]}" >> "$COMMAND_LOG" + echo "$masked_args" >> "$COMMAND_LOG" fi if [ -n "$COVERAGE_LOG" ]; then - record_command_v2 "${masked_args[*]}" + record_command_v2 "$masked_args" fi fi local command_result=0 diff --git a/tests/drivers/list_buckets/list_buckets_rest.sh b/tests/drivers/list_buckets/list_buckets_rest.sh index e4cbfa90..cac1d7e8 100644 --- a/tests/drivers/list_buckets/list_buckets_rest.sh +++ b/tests/drivers/list_buckets/list_buckets_rest.sh @@ -14,6 +14,8 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/list_buckets.sh + bucket_exists_in_list() { if ! check_param_count_gt "bucket, buckets in list" 2 $#; then return 1 diff --git a/tests/drivers/openssl.sh b/tests/drivers/openssl.sh index 89ba2ade..c974b3ac 100644 --- a/tests/drivers/openssl.sh +++ b/tests/drivers/openssl.sh @@ -21,76 +21,106 @@ write_openssl_command_to_command_log() { if ! check_param_count_v2 "command file" 1 $#; then return 1 fi - max_chars=1024 + + local max_chars=1024 response file_size log_data + if [ -n "$MAX_OPENSSL_COMMAND_LOG_BYTES" ]; then max_chars="$MAX_OPENSSL_COMMAND_LOG_BYTES" fi - if ! file_size=$(get_file_size "$1"); then + + if ! response=$(get_file_size "$1" 2>&1); then + log 2 "error getting file size: $response" return 1 fi + file_size="$response" + if [ "$max_chars" -eq -1 ] || [ "$file_size" -lt "$max_chars" ]; then log_data=$(perl -pe 's/\x00//g' "$1" | perl -pe 's/\r//g') else log_data=$(head -c "$max_chars" "$1" | perl -pe 's/\x00//g' | perl -pe 's/\r//g') log_data+="" fi + while IFS=$' ' read -r -a line_words; do - if ! mask_arg_array "${line_words[@]}"; then + if [ "${#line_words[@]}" -eq 0 ]; then + masked_words="" + elif ! response=$(mask_arg_array "${line_words[@]}" 2>&1); then + echo "error masking args: $response" return 1 + else + masked_words="$response" fi # shellcheck disable=SC2154 - echo "${masked_args[*]}" >> "$COMMAND_LOG" + echo "$masked_words" >> "$COMMAND_LOG" done <<< "$log_data" + return 0 } send_via_openssl() { if ! check_param_count_v2 "command file" 1 $#; then return 1 fi + local host response + host="${AWS_ENDPOINT_URL#http*://}" if [[ "$host" =~ s3\..*amazonaws\.com ]]; then host+=":443" fi log 5 "connecting to $host" if [ -n "$COMMAND_LOG" ]; then - write_openssl_command_to_command_log "$1" + if ! response=$(write_openssl_command_to_command_log "$1" 2>&1); then + log 2 "error writing OpenSSL command to command log: $response" + return 1 + fi fi if ! record_openssl_command "$1"; then log 3 "error recording openssl command" fi log 5 "sending openssl command file '$1'" - if ! result=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then - log 2 "error sending openssl command: $result" + if ! response=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then + log 2 "error sending openssl command: $response" return 1 fi - echo "$result" + log 5 "response: $response" + + echo "$response" + return 0 } send_via_openssl_and_check_code() { if ! check_param_count_v2 "command file, expected code" 2 $#; then return 1 fi - if ! result=$(send_via_openssl "$1"); then - log 2 "error sending command via openssl" + + local response http_response response_code + if ! response=$(send_via_openssl "$1" 2>&1); then + log 2 "error sending command via openssl: $response" return 1 fi - response_code="$(echo "$result" | grep "HTTP/" | awk '{print $2}')" + http_response="$response" + + response_code="$(echo "$http_response" | grep "HTTP/" | awk '{print $2}')" if [ "$response_code" != "$2" ]; then - log 2 "expected '$2', actual '$response_code' (error response: '$result')" + log 2 "expected '$2', actual '$response_code' (error response: '$http_response')" return 1 fi - echo "$result" + echo "$http_response" + return 0 } send_via_openssl_and_check_code_header() { if ! check_param_count_v2 "command file, expected code, header key, expected value" 4 $#; then return 1 fi - if ! send_via_openssl_and_check_code "$1" "$2"; then - log 2 "error sending via openssl and checking code" + local response http_response header_line header_value + + if ! response=$(send_via_openssl_and_check_code "$1" "$2" 2>&1); then + log 2 "error sending via openssl and checking code: $response" return 1 fi - header_line="$(echo "$result" | grep "$3")" + http_response="$response" + + header_line="$(echo "$http_response" | grep "$3")" if [ "$header_line" == "" ]; then log 2 "header key '$3' not found in header data" return 1 @@ -107,16 +137,24 @@ send_via_openssl_check_code_error_contains() { if ! check_param_count_v2 "command file, expected code, error, message" 4 $#; then return 1 fi - if ! result=$(send_via_openssl_and_check_code "$1" "$2"); then - log 2 "error sending and checking code" + local response response_file xml_file + + if ! response=$(get_file_names 2 2>&1); then + log 2 "error getting file names: $response" return 1 fi - echo -n "$result" > "$TEST_FILE_FOLDER/result.txt" - if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error_data.txt"; then + read -r response_file xml_file <<< "$response" + + if ! response=$(send_via_openssl_and_check_code "$1" "$2" 2>&1); then + log 2 "error sending and checking code: $response" + return 1 + fi + echo -n "$response" > "$TEST_FILE_FOLDER/$response_file" + if ! get_xml_data "$TEST_FILE_FOLDER/$response_file" "$TEST_FILE_FOLDER/$xml_file"; then log 2 "error parsing XML data from result" return 1 fi - if ! check_xml_error_contains "$TEST_FILE_FOLDER/error_data.txt" "$3" "$4"; then + if ! check_xml_error_contains "$TEST_FILE_FOLDER/$xml_file" "$3" "$4"; then log 2 "error checking xml error, message" return 1 fi @@ -127,21 +165,24 @@ send_via_openssl_with_timeout() { if ! check_param_count_v2 "command file" 1 $#; then return 1 fi + local host response exit_code=0 http_response + host="${AWS_ENDPOINT_URL#http*://}" if [[ "$host" =~ s3\..*amazonaws\.com ]]; then host+=":443" fi log 5 "connecting to $host" - local exit_code=0 - result=$(timeout 65 openssl s_client -connect "$host" -ign_eof < "$1" 2>&1) || exit_code=$? + response=$(timeout 65 openssl s_client -connect "$host" -ign_eof < "$1" 2>&1) || exit_code=$? if [ "$exit_code" == 124 ]; then log 2 "error: openssl command timed out" return 1 elif [ "$exit_code" != 0 ]; then - log 2 "error sending openssl command: exit code $exit_code, $result" + log 2 "error sending openssl command: exit code $exit_code, response: $response" return 1 fi - if ! [[ "$result" =~ .*$'\nclosed' ]]; then + http_response="$response" + + if ! [[ "$http_response" =~ .*$'\nclosed' ]]; then log 2 "connection not closed properly: $result" return 1 fi @@ -168,12 +209,20 @@ send_openssl_go_command() { if ! check_param_count_gt "expected HTTP code, params" 2 $#; then return 1 fi - if ! result=$(go run "./tests/rest_scripts/generateCommand.go" "-awsAccessKeyId" "$AWS_ACCESS_KEY_ID" "-awsSecretAccessKey" "$AWS_SECRET_ACCESS_KEY" "-awsRegion" "$AWS_REGION" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/openssl_command.txt" "${@:2}" 2>&1); then - log 2 "error sending go command and checking error: $result" + local response openssl_file + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting response file name: $response" return 1 fi - if ! result=$(send_via_openssl_and_check_code "$TEST_FILE_FOLDER/openssl_command.txt" "$1" 2>&1); then - log 2 "error sending via openssl and checking code: $result" + openssl_file="$response" + + if ! response=$(go run "./tests/rest_scripts/generateCommand.go" "-awsAccessKeyId" "$AWS_ACCESS_KEY_ID" "-awsSecretAccessKey" "$AWS_SECRET_ACCESS_KEY" "-awsRegion" "$AWS_REGION" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/$openssl_file" "${@:2}" 2>&1); then + log 2 "error sending go command and checking error: $response" + return 1 + fi + if ! response=$(send_via_openssl_and_check_code "$TEST_FILE_FOLDER/$openssl_file" "$1" 2>&1); then + log 2 "error sending via openssl and checking code: $response" return 1 fi return 0 @@ -183,12 +232,20 @@ send_openssl_go_command_check_header() { if ! check_param_count_gt "expected HTTP code, header key, value, params" 4 $#; then return 1 fi + local response openssl_file + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting response file name: $response" + return 1 + fi + openssl_file="$response" + if ! result=$(go run "./tests/rest_scripts/generateCommand.go" "-awsAccessKeyId" "$AWS_ACCESS_KEY_ID" "-awsSecretAccessKey" "$AWS_SECRET_ACCESS_KEY" \ - "-awsRegion" "$AWS_REGION" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/openssl_command.txt" "${@:4}" 2>&1); then + "-awsRegion" "$AWS_REGION" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/$openssl_file" "${@:4}" 2>&1); then log 2 "error sending go command and checking error: $result" return 1 fi - if ! send_via_openssl_and_check_code_header "$TEST_FILE_FOLDER/openssl_command.txt" "$1" "$2" "$3"; then + if ! send_via_openssl_and_check_code_header "$TEST_FILE_FOLDER/$openssl_file" "$1" "$2" "$3"; then log 2 "error sending command, checking code and header value" return 1 fi diff --git a/tests/env.sh b/tests/env.sh index bf4cd39d..f3b42197 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -23,49 +23,62 @@ if [ -n "$BASH_VERSION" ] && [ "${BASH_VERSINFO[0]}" -lt 4 ]; then fi base_setup() { - check_env_vars + if ! check_env_vars; then + log 1 "error checking env vars" + return 1 + fi if [ "$RUN_VERSITYGW" == "true" ] && [ "$UNIT_TEST" != "true" ]; then if ! run_versity_app; then - exit 1 + log 1 "error running versitygw app" + return 1 fi fi + return 0 } setup_test_log_file() { if [ -n "$TEST_LOG_FILE" ]; then - if ! error=$(touch "$TEST_LOG_FILE.tmp" 2>&1); then - log 2 "error creating log file: $error" - exit 1 + if ! error=$(touch "$TEST_LOG_FILE.$TEST_ID" 2>&1); then + log 1 "error creating test log file: $error" + return 1 fi export TEST_LOG_FILE fi + return 0 } -remove_test_log_file_if_desired() { +remove_test_file_folder_if_desired() { if [ "$REMOVE_TEST_FILE_FOLDER" == "true" ]; then log 6 "removing test file folder" if ! error=$(rm -rf "${TEST_FILE_FOLDER:?}" 2>&1); then log 3 "unable to remove test file folder: $error" fi fi + return 0 } check_env_vars() { - check_universal_vars + if ! check_universal_vars; then + log 1 "error checking universal env vars" + return 1 + fi if [[ $RUN_VERSITYGW == "true" ]]; then - check_versity_vars + if ! check_versity_vars; then + log 1 "error checking versitygw-related env vars" + return 1 + fi fi if [[ $RUN_S3CMD == "true" ]]; then if [ -z "$S3CMD_CONFIG" ]; then log 1 "S3CMD_CONFIG param missing" - exit 1 + return 1 fi export S3CMD_CONFIG fi if [[ $RUN_MC == "true" ]]; then if [ -z "$MC_ALIAS" ]; then log 1 "MC_ALIAS param missing" - exit 1 + return 1 fi export MC_ALIAS fi @@ -77,35 +90,36 @@ source_config_file() { if [ -r tests/.env ]; then source tests/.env else - log 3 "Warning: no .env file found in tests folder" + echo "Warning: no .env file found in tests folder" > /dev/stderr fi else # shellcheck source=./tests/.env.default source "$VERSITYGW_TEST_ENV" fi + return 0 } check_aws_vars() { if [ -z "$AWS_ACCESS_KEY_ID" ]; then log 1 "AWS_ACCESS_KEY_ID missing" - exit 1 + return 1 fi if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then log 1 "AWS_SECRET_ACCESS_KEY missing" - exit 1 + return 1 fi if [ -z "$AWS_REGION" ]; then log 1 "AWS_REGION missing" - exit 1 + return 1 fi if [ -z "$AWS_PROFILE" ]; then log 1 "AWS_PROFILE missing" - exit 1 + return 1 fi if [ "$DIRECT" != "true" ]; then if [ -z "$AWS_ENDPOINT_URL" ]; then log 1 "AWS_ENDPOINT_URL missing" - exit 1 + return 1 fi export SERVER_NAME="versitygw" else @@ -123,29 +137,31 @@ check_aws_vars() { if [ -n "$AWS_CANONICAL_ID" ]; then export AWS_CANONICAL_ID fi + return 0 } check_bucket_vars() { if [ -z "$BUCKET_ONE_NAME" ]; then log 1 "BUCKET_ONE_NAME missing" - exit 1 + return 1 fi if [ -z "$BUCKET_TWO_NAME" ]; then log 1 "BUCKET_TWO_NAME missing" - exit 1 + return 1 fi if [ "$RECREATE_BUCKETS" != "true" ] && [ "$RECREATE_BUCKETS" != "false" ]; then log 1 "RECREATE_BUCKETS must be 'true' or 'false'" - exit 1 + return 1 fi if [ "$DELETE_BUCKETS_AFTER_TEST" != "true" ] && [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ]; then log 1 "DELETE_BUCKETS_AFTER_TEST must be 'true' or 'false'" - exit 1 + return 1 fi if [ "$RECREATE_BUCKETS" == "false" ] && [ "$DELETE_BUCKETS_AFTER_TEST" == "true" ]; then log 1 "cannot set DELETE_BUCKETS_AFTER_TEST to 'true' if RECREATE_BUCKETS is 'false'" - exit 1 + return 1 fi + return 0 } check_universal_vars() { @@ -153,7 +169,10 @@ check_universal_vars() { source_config_file fi if [ -n "$COMMAND_LOG" ]; then - init_command_log + if ! init_command_log; then + log 1 "error initializing command log" + return 1 + fi fi if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then # shellcheck source=./tests/.secrets @@ -164,7 +183,7 @@ check_universal_vars() { if [[ -n "$LOG_LEVEL" ]]; then if [[ $LOG_LEVEL -lt 2 ]]; then log 1 "log level must be 2 or greater" - exit 1 + return 1 fi export LOG_LEVEL_INT=$LOG_LEVEL fi @@ -181,53 +200,44 @@ check_universal_vars() { export COVERAGE_LOG fi - check_aws_vars + if ! check_aws_vars; then + log 1 "error checking AWS-related env vars" + return 1 + fi if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then log 1 "RUN_VERSITYGW must be 'true' or 'false'" - exit 1 + return 1 fi - check_bucket_vars + if ! check_bucket_vars; then + log 1 "error checking bucket-related env vars" + return 1 + fi if [ -z "$TEST_FILE_FOLDER" ]; then log 1 "TEST_FILE_FOLDER missing" - exit 1 + return 1 fi if [ ! -d "$TEST_FILE_FOLDER" ]; then if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then log 1 "error creating test folder: $error" - exit 1 - fi - fi - export TEST_FILE_FOLDER -} - -delete_command_log() { - if [ -e "$COMMAND_LOG" ]; then - if ! error=$(rm "$COMMAND_LOG"); then - log 2 "error removing command log: $error" return 1 fi fi -} - -init_command_log() { - if ! delete_command_log; then - exit 1 - fi - echo "******** $(date +"%Y-%m-%d %H:%M:%S") $BATS_TEST_NAME COMMANDS ********" >> "$COMMAND_LOG" + export TEST_FILE_FOLDER + return 0 } check_versity_vars() { if [ -z "$LOCAL_FOLDER" ]; then log 1 "LOCAL_FOLDER missing" - exit 1 + return 1 fi if [ ! -d "$LOCAL_FOLDER" ]; then if ! error=$(mkdir -p "$LOCAL_FOLDER"); then log 2 "error creating local posix folder: $error" - exit 1 + return 1 fi fi if [ -n "$VERSIONING_DIR" ] && [ ! -d "$VERSIONING_DIR" ]; then @@ -238,51 +248,53 @@ check_versity_vars() { fi if [ -z "$VERSITY_EXE" ]; then log 1 "VERSITY_EXE missing" - exit 1 + return 1 fi if [ -z "$BACKEND" ]; then log 1 "BACKEND missing" - exit 1 + return 1 fi export LOCAL_FOLDER VERSITY_EXE BACKEND if [ "$BACKEND" == 's3' ]; then if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then log 1 "AWS_ACCESS_KEY_ID_TWO missing" - exit 1 + return 1 fi if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then log 1 "AWS_SECRET_ACCESS_KEY_TWO missing" - exit 1 + return 1 fi export AWS_ACCESS_KEY_ID_TWO AWS_SECRET_ACCESS_KEY_TWO fi - if [[ -r $GOCOVERDIR ]]; then + if [[ -n $GOCOVERDIR ]]; then export GOCOVERDIR=$GOCOVERDIR fi - if [[ $RUN_USERS == "true" ]]; then - check_user_vars + if [[ $RUN_USERS == "true" ]] && ! check_user_vars; then + log 2 "error checking versitygw users env vars" + return 1 fi + return 0 } check_user_vars() { if [ -z "$USERNAME_ONE" ]; then log 1 "USERNAME_ONE missing" - exit 1 + return 1 fi if [ -z "$PASSWORD_ONE" ]; then log 1 "PASSWORD_ONE missing" - exit 1 + return 1 fi if [ -z "$USERNAME_TWO" ]; then log 1 "USERNAME_TWO missing" - exit 1 + return 1 fi if [ -z "$PASSWORD_TWO" ]; then log 1 "PASSWORD_TWO missing" - exit 1 + return 1 fi if [ "$AUTOGENERATE_USERS" != "true" ] && [ "$AUTOGENERATE_USERS" != "false" ]; then log 1 "AUTOGENERATE_USERS must be 'true' or 'false'" @@ -303,12 +315,12 @@ check_user_vars() { if [[ "$IAM_TYPE" == "folder" ]]; then if [ -z "$USERS_FOLDER" ]; then log 1 "USERS_FOLDER missing" - exit 1 + return 1 fi if [ ! -d "$USERS_FOLDER" ]; then if ! mkdir_error=$(mkdir "$USERS_FOLDER" 2>&1); then log 1 "error creating users folder: $mkdir_error" - exit 1 + return 1 fi fi IAM_PARAMS="--iam-dir=$USERS_FOLDER" @@ -318,7 +330,7 @@ check_user_vars() { if [[ $IAM_TYPE == "s3" ]]; then if [ -z "$USERS_BUCKET" ]; then log 1 "error creating USERS_BUCKET" - exit 1 + return 1 fi IAM_PARAMS="--s3-iam-access $AWS_ACCESS_KEY_ID --s3-iam-secret $AWS_SECRET_ACCESS_KEY \ --s3-iam-region $AWS_REGION --s3-iam-bucket $USERS_BUCKET --s3-iam-endpoint $AWS_ENDPOINT_URL \ @@ -327,26 +339,109 @@ check_user_vars() { return 0 fi log 1 "unrecognized IAM_TYPE value: $IAM_TYPE" - exit 1 + return 1 } -log_cleanup() { - if [ -e "$TEST_LOG_FILE.tmp" ]; then - if ! error=$(cat "$TEST_LOG_FILE.tmp" >> "$TEST_LOG_FILE" 2>&1); then - log 3 "error appending temp log to main log: $error" - fi - if ! delete_temp_log_if_exists; then - log 3 "error deleting temp log" +delete_command_log() { + if [ -f "$COMMAND_LOG" ]; then + if ! error=$(rm "$COMMAND_LOG"); then + log 2 "error removing command log: $error" + return 1 fi fi + return 0 } -delete_temp_log_if_exists() { - if [ -e "$TEST_LOG_FILE.tmp" ]; then - if ! error=$(rm "$TEST_LOG_FILE.tmp" 2>&1); then +init_command_log() { + if ! delete_command_log; then + log 1 "error deleting old command log" + return 1 + fi + if ! echo "******** $(date +"%Y-%m-%d %H:%M:%S") $BATS_TEST_NAME COMMANDS ********" >> "$COMMAND_LOG"; then + log 1 "fatal error: unable to write to file '$COMMAND_LOG'" + return 1 + fi + return 0 +} + +main_log_cleanup() { + if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then + if ! error=$(cat "${TEST_LOG_FILE}.${TEST_ID}" >> "$TEST_LOG_FILE" 2>&1); then + log 2 "error appending temp log to main log: $error" + fi + if ! error=$(rm "${TEST_LOG_FILE}.${TEST_ID}" 2>&1); then log 2 "error deleting temp log: $error" return 1 fi fi return 0 } + +teardown_logs() { + local response + + if [[ $LOG_LEVEL -ge 4 ]] || [[ -n "$TIME_LOG" ]]; then + teardown_time_log + fi + if [[ -f "${TEST_LOG_FILE}.${TEST_ID}" ]]; then + echo "********************************** END TEST LOG **********************************" >> "${TEST_LOG_FILE}.${TEST_ID}" + fi + if [[ -f "$COMMAND_LOG" ]]; then + teardown_command_log + fi + if [ -f "${VERSITY_LOG_FILE}.${TEST_ID}.1" ]; then + teardown_versity_log 1 + fi + if [ -f "${VERSITY_LOG_FILE}.${TEST_ID}.2" ]; then + teardown_versity_log 2 + fi + if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ] && [ "$BATS_TEST_COMPLETED" != "1" ]; then + cat "${TEST_LOG_FILE}.${TEST_ID}" + fi + main_log_cleanup +} + +teardown_time_log() { + local end_time total_time + + end_time=$(date +%s) + total_time=$((end_time - START_TIME)) + log 4 "Total test time: $total_time" + if [[ -n "$TIME_LOG" ]]; then + if ! echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"; then + log 3 "unable to write to '$TIME_LOG', check permissions" + return 1 + fi + fi + return 0 +} + +teardown_command_log() { + echo "**********************************************************************************" >> "$COMMAND_LOG" + if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then + cat "$COMMAND_LOG" >> "${TEST_LOG_FILE}.${TEST_ID}" + elif [ "$BATS_TEST_COMPLETED" != "1" ]; then + cat "$COMMAND_LOG" + fi + if ! delete_command_log; then + log 2 "error deleting command log" + return 1 + fi + return 0 +} + +teardown_versity_log() { + if ! check_param_count_v2 "versitygw process ID" 1 $#; then + return 1 + fi + echo "**********************************************************************************" >> "${VERSITY_LOG_FILE}.${TEST_ID}.$1" + if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then + cat "${VERSITY_LOG_FILE}.${TEST_ID}.$1" >> "${TEST_LOG_FILE}.${TEST_ID}" + elif [ "$BATS_TEST_COMPLETED" != "1" ]; then + cat "${VERSITY_LOG_FILE}.${TEST_ID}.$1" + fi + if ! response=$(rm "${VERSITY_LOG_FILE}.${TEST_ID}.$1" 2>&1); then + log 3 "error deleting log file '${VERSITY_LOG_FILE}.${TEST_ID}.$1': $response" + fi + return 0 +} diff --git a/tests/logger.sh b/tests/logger.sh index df3d93cb..8dc9b7cf 100644 --- a/tests/logger.sh +++ b/tests/logger.sh @@ -62,27 +62,36 @@ log_with_stack_ref() { log_mask "$log_level" "$2" "$3" return 0 fi - log_message "$log_level" "$2" "$3" + if ! log_message "$log_level" "$2" "$3"; then + return 1 + fi + return 0 } log_mask() { if ! check_log_params "log_mask" "level, string, stack reference" 3 $#; then return 1 fi - if ! mask_args "$2"; then - echo "error masking args" + local response masked_data + + if ! response=$(mask_args "$2" 2>&1); then + echo "error masking args: $response" return 1 fi + masked_data="$response" - log_message "$log_level" "$masked_data" "$3" + if ! log_message "$1" "$masked_data" "$3"; then + return 1 + fi + return 0 } mask_args() { if ! check_log_params "mask_args" "string" 1 $#; then return 1 fi - unmasked_array=() - masked_data="" + local unmasked_array=() first_line masked_data="" response masked_line + while IFS= read -r line; do unmasked_array+=("$line") done <<< "$1" @@ -90,60 +99,69 @@ mask_args() { # shellcheck disable=SC2068 first_line=true for line in "${unmasked_array[@]}"; do - if ! mask_arg_array "$line"; then - echo "error masking arg array" + if ! response=$(mask_arg_array "$line" 2>&1); then + echo "error masking arg array: $response" return 1 fi + masked_line="$response" + if [ "$first_line" == "true" ]; then - masked_data="${masked_args[*]}" + masked_data="$masked_line" first_line="false" else - masked_data+=$(printf "\n%s" "${masked_args[*]}") + masked_data+=$(printf "\n%s" "$masked_line") fi done + echo "$masked_data" + return 0 } mask_arg_array() { if [ $# -eq 0 ]; then - echo "'mask_arg_array' requires parameters" + echo "'mask_arg_array' requires data to be logged" return 1 fi - mask_next=false - is_access=false - masked_args=() # Initialize an array to hold the masked arguments + + local mask_next=false is_access=false masked_args=() # Initialize an array to hold the masked arguments # shellcheck disable=SC2068 for arg in $@; do - if ! check_arg_for_mask "$arg"; then - echo "error checking arg for mask" + if ! response=$(check_arg_for_mask "$mask_next" "$is_access" "$arg" 2>&1); then + echo "error checking arg for mask: $response" return 1 fi + read -r mask_next is_access masked_arg <<< "$response" + masked_args+=("$masked_arg") done + echo "${masked_args[*]}" + return 0 } check_arg_for_mask() { - if ! check_log_params "check_arg_for_mask" "arg" 1 $#; then + if ! check_log_params "check_arg_for_mask" "mask next, is access ID, arg" 3 $#; then return 1 fi + local mask_next="$1" is_access="$2" arg="$3" + if [[ $mask_next == true ]]; then if [ "$is_access" == "true" ]; then - masked_args+=("${arg:0:4}****") + masked_arg="${arg:0:4}****" is_access=false else - masked_args+=("\********") + masked_arg="\********" fi mask_next=false elif [[ "$arg" == --secret_key=* ]]; then - masked_args+=("--secret_key=********") + masked_arg="--secret_key=********" elif [[ "$arg" == --secret=* ]]; then - masked_args+=("--secret=********") + masked_arg="--secret=********" elif [[ "$arg" == --access=* ]]; then - masked_args+=("${arg:0:13}****") + masked_arg="${arg:0:13}****" elif [[ "$arg" == --access_key=* ]]; then - masked_args+=("${arg:0:17}****") + masked_arg="${arg:0:17}****" elif [[ "$arg" == *"Credential="* ]]; then - masked_args+=("$(echo "$arg" | sed -E 's/(Credential=[A-Z0-9]{5})[^\/]*/\1****/g')") + masked_arg="$(echo "$arg" | sed -E 's/(Credential=[A-Z0-9]{5})[^\/]*/\1****/g')" elif [[ "$arg" == *"AWS_ACCESS_KEY_ID="* ]]; then - masked_args+=("AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:0:5}****") + masked_arg="AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:0:5}****" else if [[ "$arg" == "--secret_key" ]] || [[ "$arg" == "--secret" ]] || [[ "$arg" == "--s3-iam-secret" ]]; then mask_next=true @@ -151,21 +169,26 @@ check_arg_for_mask() { mask_next=true is_access=true fi - masked_args+=("$arg") + masked_arg="$arg" fi + echo "$mask_next $is_access $masked_arg" + return 0 } log_message() { if ! check_log_params "log_message" "level, message, stack reference" 3 $#; then return 1 fi - local bash_source_ref=$(($3+1)) + local bash_source_ref=$(($3+1)) now now="$(date "+%Y-%m-%d %H:%M:%S")" if [[ ( "$1" == "CRIT" ) || ( "$1" == "ERROR" ) ]]; then printf "%s\n" "$now $1 $2" >&2 fi if [[ -n "$TEST_LOG_FILE" ]]; then - printf "%s\n" "$now ${BASH_SOURCE[$bash_source_ref]}:${BASH_LINENO[$3]} $1 $2" >> "$TEST_LOG_FILE.tmp" + if ! printf "%s\n" "$now ${BASH_SOURCE[$bash_source_ref]}:${BASH_LINENO[$3]} $1 $2" >> "${TEST_LOG_FILE}.${TEST_ID}"; then + echo "unable to write to log file '${TEST_LOG_FILE}.${TEST_ID}'" + return 1 + fi fi - sync + return 0 } diff --git a/tests/report.sh b/tests/report.sh index cb91a5d8..69da0953 100644 --- a/tests/report.sh +++ b/tests/report.sh @@ -194,6 +194,8 @@ write_to_coverage_log() { if ! check_param_count_v2 "string" 1 $#; then return 1 fi + local response + echo "$1" >> "$COVERAGE_LOG" sort "$COVERAGE_LOG" | uniq > "${COVERAGE_LOG}.tmp" mv "${COVERAGE_LOG}.tmp" "$COVERAGE_LOG" @@ -224,14 +226,19 @@ parse_command_info() { if ! check_param_count_v2 "command string" 1 $#; then return 1 fi + local response command_info + if [[ "$1" == *"curl "* ]]; then - if ! command_info=$(parse_curl_rest_command "$1" 2>&1); then - echo "error parsing rest command: $command_info" + if ! response=$(parse_curl_rest_command "$1" 2>&1); then + echo "error parsing rest command: $response" return 1 fi + command_info="$response" else command_info="OTHER" fi + echo "$command_info" + return 0 } record_command_v2() { @@ -239,13 +246,17 @@ record_command_v2() { log 5 "no coverage log set" return 0 fi + local response command_info + if ! check_param_count_v2 "command string" 1 $#; then return 1 fi - if ! parse_command_info "$1"; then - log 2 "error parsing command info" + if ! response=$(parse_command_info "$1" 2>&1); then + log 2 "error parsing command info: $response" return 1 fi + command_info="$response" + if [ "$command_info" == "OTHER" ]; then return 0 fi @@ -253,4 +264,5 @@ record_command_v2() { log 2 "error writing to coverage log" return 1 fi + return 0 } diff --git a/tests/setup.sh b/tests/setup.sh index c89732eb..e8a98c2f 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -64,9 +64,24 @@ static_user_versitygw_setup() { # bats setup function setup() { - base_setup + TEST_ID=$(date +"%Y%m%d-%H%M%S").$(basename "$BATS_TEST_FILENAME").$BATS_TEST_NAME + export TEST_ID - setup_test_log_file + source_config_file + + if ! setup_test_log_file; then + log 1 "error creating test log file" + return 1 + fi + + if [ -n "$TEST_LOG_FILE" ]; then + printf "\n%s\n\n" "**** $TEST_ID ****" >> "${TEST_LOG_FILE}.${TEST_ID}" + fi + + if ! base_setup; then + log 2 "error checking env vars or running versitygw" + return 1 + fi if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ]; then if ! static_user_versitygw_setup; then @@ -77,8 +92,8 @@ setup() { log 4 "Running test $BATS_TEST_NAME" if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then - start_time=$(date +%s) - export start_time + START_TIME=$(date +%s) + export START_TIME fi if [[ $RUN_S3CMD == true ]]; then @@ -90,47 +105,40 @@ setup() { export S3CMD_CONFIG S3CMD_OPTS fi - if [[ $RUN_MC == true ]]; then - check_add_mc_alias + if [[ $RUN_MC == true ]] && ! check_add_mc_alias; then + log 1 "error checking for or adding mc alias" + return 1 fi export AWS_PROFILE log 4 "********** END SETUP **********" -} - -post_versity_cleanup() { - if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then - end_time=$(date +%s) - total_time=$((end_time - start_time)) - log 4 "Total test time: $total_time" - if [[ -n "$TIME_LOG" ]]; then - echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG" - fi - fi - if [[ "$BATS_TEST_COMPLETED" -ne 1 ]]; then - if [[ -e "$COMMAND_LOG" ]]; then - cat "$COMMAND_LOG" - echo "**********************************************************************************" - fi - if [[ -e "$TEST_LOG_FILE.tmp" ]]; then - echo "********************************** LOG *******************************************" - cat "$TEST_LOG_FILE.tmp" - echo "**********************************************************************************" - fi - fi - if ! delete_command_log; then - log 3 "error deleting command log" - fi - log_cleanup + return 0 } # bats teardown function teardown() { - # shellcheck disable=SC2154 + local response proc_status_one proc_status_two + + response=$(check_versity_process_status) + read -r proc_status_one proc_status_two <<< "$response" + + if [ "$proc_status_one" == "none" ] || [[ ( "$proc_status_one" == "running" ) && (( "$proc_status_two" == "none") || ( "$proc_status_two" == "running")) ]]; then + bucket_and_user_cleanup + fi + if [ "$proc_status_one" == "running" ]; then + stop_versity_process "$VERSITYGW_PID_1" + fi + if [ "$proc_status_two" == "running" ]; then + stop_versity_process "$VERSITYGW_PID_2" + fi + remove_test_file_folder_if_desired + teardown_logs +} + +bucket_and_user_cleanup() { log 4 "********** BEGIN TEARDOWN **********" if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then log 3 "error cleaning up buckets after test" - return 1 fi if [ "$SKIP_USERS_TESTS" != "true" ]; then if [ -n "$USERNAME_ONE" ]; then @@ -147,7 +155,5 @@ teardown() { if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then log 3 "error deleting autocreated users" fi - remove_test_log_file_if_desired - stop_versity - post_versity_cleanup + return 0 } diff --git a/tests/setup_static.sh b/tests/setup_static.sh index 958e4887..8c6857ae 100755 --- a/tests/setup_static.sh +++ b/tests/setup_static.sh @@ -49,6 +49,6 @@ fi # shellcheck disable=SC2034 RECREATE_BUCKETS=false -if ! stop_versity; then +if ! stop_versity_process "$VERSITYGW_PID_1"; then log 2 "error stopping versity" fi diff --git a/tests/setup_unit.sh b/tests/setup_unit.sh index a42386dd..d6cd9007 100644 --- a/tests/setup_unit.sh +++ b/tests/setup_unit.sh @@ -24,6 +24,5 @@ setup() { } teardown() { - remove_test_log_file_if_desired - log_cleanup + teardown_logs } \ No newline at end of file diff --git a/tests/versity.sh b/tests/versity.sh index fb28cce2..87f512a4 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -14,53 +14,89 @@ # specific language governing permissions and limitations # under the License. +source ./tests/commands/list_buckets.sh + start_versity_process() { - if ! check_param_count "start_versity_process" "versity app index" 1 $#; then + if ! check_param_count_gt "versity app index, command array" 2 $#; then exit 1 fi - build_run_and_log_command - # shellcheck disable=SC2181 - if [[ $? -ne 0 ]]; then - sleep 1 - if [ -n "$VERSITY_LOG_FILE" ]; then - log 1 "error running versitygw command: $(cat "$VERSITY_LOG_FILE")" - return 1 - fi - return 1 - fi - eval versitygw_pid_"$1"=$! - if [ -n "$VERSITY_LOG_FILE" ]; then - process_info="Versity process $1, PID $!" - echo "$process_info" >> "$VERSITY_LOG_FILE" - fi - log 4 "$process_info" - local pid - eval pid=\$versitygw_pid_"$1" - sleep 1 + local response process_id - if ! check_result=$(kill -0 "$pid" 2>&1); then - log 1 "versitygw failed to start: $check_result" - if [ -n "$VERSITY_LOG_FILE" ]; then - log 1 "log data: $(cat "$VERSITY_LOG_FILE")" - fi + if ! response=$(build_run_and_log_command "$1" "${@:2}" 2>&1); then + log 2 "error building, logging, and/or running 'versitygw' executable: $response" return 1 fi - export versitygw_pid_"$1" + process_id="$response" + + printf -v "VERSITYGW_PID_$1" '%s' "$process_id" + export VERSITYGW_PID_"$1" + + return 0 } build_run_and_log_command() { - IFS=' ' read -r -a full_command <<< "${base_command[@]}" + if ! check_param_count_gt "versitygw process number (1 or 2), command array" 2 $#; then + return 1 + fi + local response full_command versitygw_log_file_name="" pid check_result + + IFS=' ' read -r -a full_command <<< "${@:2}" log 5 "versity command: ${full_command[*]}" if [ -n "$COMMAND_LOG" ]; then - mask_args "${full_command[*]}" + if ! response=$(mask_args "${full_command[*]}" 2>&1); then + log 2 "error masking versitygw command" + return 1 + fi # shellcheck disable=SC2154 - echo "${masked_args[@]}" >> "$COMMAND_LOG" + echo "$response" >> "$COMMAND_LOG" fi if [ -n "$VERSITY_LOG_FILE" ]; then - "${full_command[@]}" >> "$VERSITY_LOG_FILE" 2>&1 & + versitygw_log_file_name="$VERSITY_LOG_FILE.$TEST_ID".$1 + echo "****************************** VERSITYGW $1 LOG ***********************************" >> "$versitygw_log_file_name" + "${full_command[@]}" >> "$versitygw_log_file_name" 2>&1 & else - "${full_command[@]}" 2>&1 & + "${full_command[@]}" >/dev/null 2>&1 & fi + + pid="$!" + if ! verify_process_started "$pid" "$versitygw_log_file_name"; then + return 1 + fi + + echo "$pid" + return 0 +} + +verify_process_started() { + if ! check_param_count_ge_le "pid, log file (if any)" 1 2 $#; then + return 1 + fi + local check_result process_running="false" proc_state + + for ((check_num=1; check_num<=3; check_num++)); do + sleep 1 + if [ "$process_running" == "false" ] && check_result=$(kill -0 "$1" 2>&1); then + process_running="true" + fi + if [ "$process_running" == "true" ]; then + proc_state=$(ps -p "$1" -o state= 2>/dev/null | tr -d ' ') + if [ "$proc_state" != "Z" ] && list_buckets_rest "" "" >/dev/null; then + return 0 + fi + fi + done + + if [ "$process_running" == "false" ]; then + log 1 "versitygw failed to start or crashed: $check_result" + elif [[ "$proc_state" == "Z" ]]; then + log 1 "versitygw process running in zombie state" + else + log 1 "process running in state '$proc_state', but not communicating properly" + fi + if [[ -n "$2" ]]; then + log 1 "log data: '$(cat "$2")'" + fi + return 1 } run_versity_app_posix() { @@ -84,9 +120,11 @@ run_versity_app_posix() { base_command+=(--versioning-dir "$VERSIONING_DIR") fi base_command+=("$LOCAL_FOLDER") - export base_command - start_versity_process "$3" + if ! start_versity_process "$3" "${base_command[@]}"; then + log 1 "error starting versity process" + return 1 + fi return 0 } @@ -102,9 +140,11 @@ run_versity_app_scoutfs() { base_command+=(--port ":$PORT") fi base_command+=(scoutfs "$LOCAL_FOLDER") - export base_command - start_versity_process "$3" + if ! start_versity_process "$3" "${base_command[@]}"; then + log 1 "error starting versity process" + return 1 + fi return 0 } @@ -122,20 +162,30 @@ run_versity_app_s3() { base_command+=(--port ":7071") fi base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com) - export base_command - start_versity_process "$1" + if ! start_versity_process "$1" "${base_command[@]}"; then + log 2 "error starting versity process" + return 1 + fi return 0 } run_versity_app() { if [[ $BACKEND == 'posix' ]]; then - run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" + if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then + return 1 + fi elif [[ $BACKEND == 'scoutfs' ]]; then - run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" + if ! run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then + return 1 + fi elif [[ $BACKEND == 's3' ]]; then - run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" - run_versity_app_s3 "2" + if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then + return 1 + fi + if ! run_versity_app_s3 "2"; then + return 1 + fi else log 1 "unrecognized backend type $BACKEND" return 1 @@ -151,9 +201,10 @@ run_versity_app() { teardown return 1 fi + return 0 } -stop_single_process() { +stop_versity_process() { if ! check_param_count "stop_single_process" "versitygw PID" 1 $#; then return 1 fi @@ -161,27 +212,31 @@ stop_single_process() { # shellcheck disable=SC2086 if ps_result=$(ps -p $1 2>&1) > /dev/null; then kill "$1" - wait "$1" || true + wait "$1" 2>/dev/null || true else log 3 "error stopping versity app: $ps_result" fi } -stop_versity() { - if [ "$RUN_VERSITYGW" == "false" ]; then - return - fi - if [[ -z "$versitygw_pid_1" ]]; then - return - fi - # shellcheck disable=SC2154 - if ! stop_single_process "$versitygw_pid_1"; then - log 2 "error stopping versity process" - fi - if [[ $BACKEND == 's3' ]] && [[ -n "$versitygw_pid_2" ]]; then - # shellcheck disable=SC2154 - if ! stop_single_process "$versitygw_pid_2"; then - log 2 "error stopping versity process two" +check_versity_process_status() { + local status_one="" status_two="" + + status_one="none" + status_two="none" + if [ "$RUN_VERSITYGW" == "true" ]; then + if [[ -n "$VERSITYGW_PID_1" ]] && verify_process_started "$VERSITYGW_PID_1" >/dev/null; then + status_one="running" + else + status_one="failed" + fi + if [ "$BACKEND" == "s3" ]; then + if [ -n "$VERSITYGW_PID_2" ] && verify_process_started "$VERSITYGW_PID_2" >/dev/null; then + status_two="running" + else + status_two="failed" + fi fi fi -} \ No newline at end of file + echo "$status_one $status_two" + return 0 +}