mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-07-02 16:54:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: jailing work (base)

Browse Source
This commit is contained in:
Luke McCrone
2026-06-05 09:32:45 -03:00
parent b348ff29f0
commit f826f13a64
10 changed files with 491 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/commands/command.sh
+8 -3
View File
@@ -21,17 +21,22 @@ send_command() {
if ! check_param_count_gt "command data" 1 $#; then
return 1
fi
local response
if [ -n "$COMMAND_LOG" ] || [ -n "$COVERAGE_LOG" ]; then
args=(AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" "$@")
if ! mask_arg_array "${args[@]}"; then
if ! response=$(mask_arg_array "${args[@]}" 2>&1); then
echo "error masking args: $response"
return 1
fi
masked_args="$response"
if [ -n "$COMMAND_LOG" ]; then
# shellcheck disable=SC2154
echo "${masked_args[*]}" >> "$COMMAND_LOG"
echo "$masked_args" >> "$COMMAND_LOG"
fi
if [ -n "$COVERAGE_LOG" ]; then
record_command_v2 "${masked_args[*]}"
record_command_v2 "$masked_args"
fi
fi
local command_result=0
tests/drivers/list_buckets/list_buckets_rest.sh
+2
View File
@@ -14,6 +14,8 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/list_buckets.sh
bucket_exists_in_list() {
if ! check_param_count_gt "bucket, buckets in list" 2 $#; then
return 1
tests/drivers/openssl.sh
+88 -31
View File
@@ -21,76 +21,106 @@ write_openssl_command_to_command_log() {
if ! check_param_count_v2 "command file" 1 $#; then
return 1
fi
max_chars=1024
local max_chars=1024 response file_size log_data
if [ -n "$MAX_OPENSSL_COMMAND_LOG_BYTES" ]; then
max_chars="$MAX_OPENSSL_COMMAND_LOG_BYTES"
fi
if ! file_size=$(get_file_size "$1"); then
if ! response=$(get_file_size "$1" 2>&1); then
log 2 "error getting file size: $response"
return 1
fi
file_size="$response"
if [ "$max_chars" -eq -1 ] || [ "$file_size" -lt "$max_chars" ]; then
log_data=$(perl -pe 's/\x00/<NULL>/g' "$1" | perl -pe 's/\r/<CR>/g')
else
log_data=$(head -c "$max_chars" "$1" | perl -pe 's/\x00/<NULL>/g' | perl -pe 's/\r/<CR>/g')
log_data+="<TRUNC>"
fi
while IFS=$' ' read -r -a line_words; do
if ! mask_arg_array "${line_words[@]}"; then
if [ "${#line_words[@]}" -eq 0 ]; then
masked_words=""
elif ! response=$(mask_arg_array "${line_words[@]}" 2>&1); then
echo "error masking args: $response"
return 1
else
masked_words="$response"
fi
# shellcheck disable=SC2154
echo "${masked_args[*]}" >> "$COMMAND_LOG"
echo "$masked_words" >> "$COMMAND_LOG"
done <<< "$log_data"
return 0
}
send_via_openssl() {
if ! check_param_count_v2 "command file" 1 $#; then
return 1
fi
local host response
host="${AWS_ENDPOINT_URL#http*://}"
if [[ "$host" =~ s3\..*amazonaws\.com ]]; then
host+=":443"
fi
log 5 "connecting to $host"
if [ -n "$COMMAND_LOG" ]; then
write_openssl_command_to_command_log "$1"
if ! response=$(write_openssl_command_to_command_log "$1" 2>&1); then
log 2 "error writing OpenSSL command to command log: $response"
return 1
fi
fi
if ! record_openssl_command "$1"; then
log 3 "error recording openssl command"
fi
log 5 "sending openssl command file '$1'"
if ! result=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then
log 2 "error sending openssl command: $result"
if ! response=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then
log 2 "error sending openssl command: $response"
return 1
fi
echo "$result"
log 5 "response: $response"
echo "$response"
return 0
}
send_via_openssl_and_check_code() {
if ! check_param_count_v2 "command file, expected code" 2 $#; then
return 1
fi
if ! result=$(send_via_openssl "$1"); then
log 2 "error sending command via openssl"
local response http_response response_code
if ! response=$(send_via_openssl "$1" 2>&1); then
log 2 "error sending command via openssl: $response"
return 1
fi
response_code="$(echo "$result" | grep "HTTP/" | awk '{print $2}')"
http_response="$response"
response_code="$(echo "$http_response" | grep "HTTP/" | awk '{print $2}')"
if [ "$response_code" != "$2" ]; then
log 2 "expected '$2', actual '$response_code' (error response: '$result')"
log 2 "expected '$2', actual '$response_code' (error response: '$http_response')"
return 1
fi
echo "$result"
echo "$http_response"
return 0
}
send_via_openssl_and_check_code_header() {
if ! check_param_count_v2 "command file, expected code, header key, expected value" 4 $#; then
return 1
fi
if ! send_via_openssl_and_check_code "$1" "$2"; then
log 2 "error sending via openssl and checking code"
local response http_response header_line header_value
if ! response=$(send_via_openssl_and_check_code "$1" "$2" 2>&1); then
log 2 "error sending via openssl and checking code: $response"
return 1
fi
header_line="$(echo "$result" | grep "$3")"
http_response="$response"
header_line="$(echo "$http_response" | grep "$3")"
if [ "$header_line" == "" ]; then
log 2 "header key '$3' not found in header data"
return 1
@@ -107,16 +137,24 @@ send_via_openssl_check_code_error_contains() {
if ! check_param_count_v2 "command file, expected code, error, message" 4 $#; then
return 1
fi
if ! result=$(send_via_openssl_and_check_code "$1" "$2"); then
log 2 "error sending and checking code"
local response response_file xml_file
if ! response=$(get_file_names 2 2>&1); then
log 2 "error getting file names: $response"
return 1
fi
echo -n "$result" > "$TEST_FILE_FOLDER/result.txt"
if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error_data.txt"; then
read -r response_file xml_file <<< "$response"
if ! response=$(send_via_openssl_and_check_code "$1" "$2" 2>&1); then
log 2 "error sending and checking code: $response"
return 1
fi
echo -n "$response" > "$TEST_FILE_FOLDER/$response_file"
if ! get_xml_data "$TEST_FILE_FOLDER/$response_file" "$TEST_FILE_FOLDER/$xml_file"; then
log 2 "error parsing XML data from result"
return 1
fi
if ! check_xml_error_contains "$TEST_FILE_FOLDER/error_data.txt" "$3" "$4"; then
if ! check_xml_error_contains "$TEST_FILE_FOLDER/$xml_file" "$3" "$4"; then
log 2 "error checking xml error, message"
return 1
fi
@@ -127,21 +165,24 @@ send_via_openssl_with_timeout() {
if ! check_param_count_v2 "command file" 1 $#; then
return 1
fi
local host response exit_code=0 http_response
host="${AWS_ENDPOINT_URL#http*://}"
if [[ "$host" =~ s3\..*amazonaws\.com ]]; then
host+=":443"
fi
log 5 "connecting to $host"
local exit_code=0
result=$(timeout 65 openssl s_client -connect "$host" -ign_eof < "$1" 2>&1) || exit_code=$?
response=$(timeout 65 openssl s_client -connect "$host" -ign_eof < "$1" 2>&1) || exit_code=$?
if [ "$exit_code" == 124 ]; then
log 2 "error: openssl command timed out"
return 1
elif [ "$exit_code" != 0 ]; then
log 2 "error sending openssl command: exit code $exit_code, $result"
log 2 "error sending openssl command: exit code $exit_code, response: $response"
return 1
fi
if ! [[ "$result" =~ .*$'\nclosed' ]]; then
http_response="$response"
if ! [[ "$http_response" =~ .*$'\nclosed' ]]; then
log 2 "connection not closed properly: $result"
return 1
fi
@@ -168,12 +209,20 @@ send_openssl_go_command() {
if ! check_param_count_gt "expected HTTP code, params" 2 $#; then
return 1
fi
if ! result=$(go run "./tests/rest_scripts/generateCommand.go" "-awsAccessKeyId" "$AWS_ACCESS_KEY_ID" "-awsSecretAccessKey" "$AWS_SECRET_ACCESS_KEY" "-awsRegion" "$AWS_REGION" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/openssl_command.txt" "${@:2}" 2>&1); then
log 2 "error sending go command and checking error: $result"
local response openssl_file
if ! response=$(get_file_name 2>&1); then
log 2 "error getting response file name: $response"
return 1
fi
if ! result=$(send_via_openssl_and_check_code "$TEST_FILE_FOLDER/openssl_command.txt" "$1" 2>&1); then
log 2 "error sending via openssl and checking code: $result"
openssl_file="$response"
if ! response=$(go run "./tests/rest_scripts/generateCommand.go" "-awsAccessKeyId" "$AWS_ACCESS_KEY_ID" "-awsSecretAccessKey" "$AWS_SECRET_ACCESS_KEY" "-awsRegion" "$AWS_REGION" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/$openssl_file" "${@:2}" 2>&1); then
log 2 "error sending go command and checking error: $response"
return 1
fi
if ! response=$(send_via_openssl_and_check_code "$TEST_FILE_FOLDER/$openssl_file" "$1" 2>&1); then
log 2 "error sending via openssl and checking code: $response"
return 1
fi
return 0
@@ -183,12 +232,20 @@ send_openssl_go_command_check_header() {
if ! check_param_count_gt "expected HTTP code, header key, value, params" 4 $#; then
return 1
fi
local response openssl_file
if ! response=$(get_file_name 2>&1); then
log 2 "error getting response file name: $response"
return 1
fi
openssl_file="$response"
if ! result=$(go run "./tests/rest_scripts/generateCommand.go" "-awsAccessKeyId" "$AWS_ACCESS_KEY_ID" "-awsSecretAccessKey" "$AWS_SECRET_ACCESS_KEY" \
"-awsRegion" "$AWS_REGION" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/openssl_command.txt" "${@:4}" 2>&1); then
"-awsRegion" "$AWS_REGION" "-url" "$AWS_ENDPOINT_URL" "-client" "openssl" "-filePath" "$TEST_FILE_FOLDER/$openssl_file" "${@:4}" 2>&1); then
log 2 "error sending go command and checking error: $result"
return 1
fi
if ! send_via_openssl_and_check_code_header "$TEST_FILE_FOLDER/openssl_command.txt" "$1" "$2" "$3"; then
if ! send_via_openssl_and_check_code_header "$TEST_FILE_FOLDER/$openssl_file" "$1" "$2" "$3"; then
log 2 "error sending command, checking code and header value"
return 1
fi
tests/env.sh
+166 -71
View File
@@ -23,49 +23,62 @@ if [ -n "$BASH_VERSION" ] && [ "${BASH_VERSINFO[0]}" -lt 4 ]; then
fi
base_setup() {
check_env_vars
if ! check_env_vars; then
log 1 "error checking env vars"
return 1
fi
if [ "$RUN_VERSITYGW" == "true" ] && [ "$UNIT_TEST" != "true" ]; then
if ! run_versity_app; then
exit 1
log 1 "error running versitygw app"
return 1
fi
fi
return 0
}
setup_test_log_file() {
if [ -n "$TEST_LOG_FILE" ]; then
if ! error=$(touch "$TEST_LOG_FILE.tmp" 2>&1); then
log 2 "error creating log file: $error"
exit 1
if ! error=$(touch "$TEST_LOG_FILE.$TEST_ID" 2>&1); then
log 1 "error creating test log file: $error"
return 1
fi
export TEST_LOG_FILE
fi
return 0
}
remove_test_log_file_if_desired() {
remove_test_file_folder_if_desired() {
if [ "$REMOVE_TEST_FILE_FOLDER" == "true" ]; then
log 6 "removing test file folder"
if ! error=$(rm -rf "${TEST_FILE_FOLDER:?}" 2>&1); then
log 3 "unable to remove test file folder: $error"
fi
fi
return 0
}
check_env_vars() {
check_universal_vars
if ! check_universal_vars; then
log 1 "error checking universal env vars"
return 1
fi
if [[ $RUN_VERSITYGW == "true" ]]; then
check_versity_vars
if ! check_versity_vars; then
log 1 "error checking versitygw-related env vars"
return 1
fi
fi
if [[ $RUN_S3CMD == "true" ]]; then
if [ -z "$S3CMD_CONFIG" ]; then
log 1 "S3CMD_CONFIG param missing"
exit 1
return 1
fi
export S3CMD_CONFIG
fi
if [[ $RUN_MC == "true" ]]; then
if [ -z "$MC_ALIAS" ]; then
log 1 "MC_ALIAS param missing"
exit 1
return 1
fi
export MC_ALIAS
fi
@@ -77,35 +90,36 @@ source_config_file() {
if [ -r tests/.env ]; then
source tests/.env
else
log 3 "Warning: no .env file found in tests folder"
echo "Warning: no .env file found in tests folder" > /dev/stderr
fi
else
# shellcheck source=./tests/.env.default
source "$VERSITYGW_TEST_ENV"
fi
return 0
}
check_aws_vars() {
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
log 1 "AWS_ACCESS_KEY_ID missing"
exit 1
return 1
fi
if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
log 1 "AWS_SECRET_ACCESS_KEY missing"
exit 1
return 1
fi
if [ -z "$AWS_REGION" ]; then
log 1 "AWS_REGION missing"
exit 1
return 1
fi
if [ -z "$AWS_PROFILE" ]; then
log 1 "AWS_PROFILE missing"
exit 1
return 1
fi
if [ "$DIRECT" != "true" ]; then
if [ -z "$AWS_ENDPOINT_URL" ]; then
log 1 "AWS_ENDPOINT_URL missing"
exit 1
return 1
fi
export SERVER_NAME="versitygw"
else
@@ -123,29 +137,31 @@ check_aws_vars() {
if [ -n "$AWS_CANONICAL_ID" ]; then
export AWS_CANONICAL_ID
fi
return 0
}
check_bucket_vars() {
if [ -z "$BUCKET_ONE_NAME" ]; then
log 1 "BUCKET_ONE_NAME missing"
exit 1
return 1
fi
if [ -z "$BUCKET_TWO_NAME" ]; then
log 1 "BUCKET_TWO_NAME missing"
exit 1
return 1
fi
if [ "$RECREATE_BUCKETS" != "true" ] && [ "$RECREATE_BUCKETS" != "false" ]; then
log 1 "RECREATE_BUCKETS must be 'true' or 'false'"
exit 1
return 1
fi
if [ "$DELETE_BUCKETS_AFTER_TEST" != "true" ] && [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ]; then
log 1 "DELETE_BUCKETS_AFTER_TEST must be 'true' or 'false'"
exit 1
return 1
fi
if [ "$RECREATE_BUCKETS" == "false" ] && [ "$DELETE_BUCKETS_AFTER_TEST" == "true" ]; then
log 1 "cannot set DELETE_BUCKETS_AFTER_TEST to 'true' if RECREATE_BUCKETS is 'false'"
exit 1
return 1
fi
return 0
}
check_universal_vars() {
@@ -153,7 +169,10 @@ check_universal_vars() {
source_config_file
fi
if [ -n "$COMMAND_LOG" ]; then
init_command_log
if ! init_command_log; then
log 1 "error initializing command log"
return 1
fi
fi
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
# shellcheck source=./tests/.secrets
@@ -164,7 +183,7 @@ check_universal_vars() {
if [[ -n "$LOG_LEVEL" ]]; then
if [[ $LOG_LEVEL -lt 2 ]]; then
log 1 "log level must be 2 or greater"
exit 1
return 1
fi
export LOG_LEVEL_INT=$LOG_LEVEL
fi
@@ -181,53 +200,44 @@ check_universal_vars() {
export COVERAGE_LOG
fi
check_aws_vars
if ! check_aws_vars; then
log 1 "error checking AWS-related env vars"
return 1
fi
if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
log 1 "RUN_VERSITYGW must be 'true' or 'false'"
exit 1
return 1
fi
check_bucket_vars
if ! check_bucket_vars; then
log 1 "error checking bucket-related env vars"
return 1
fi
if [ -z "$TEST_FILE_FOLDER" ]; then
log 1 "TEST_FILE_FOLDER missing"
exit 1
return 1
fi
if [ ! -d "$TEST_FILE_FOLDER" ]; then
if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then
log 1 "error creating test folder: $error"
exit 1
fi
fi
export TEST_FILE_FOLDER
}
delete_command_log() {
if [ -e "$COMMAND_LOG" ]; then
if ! error=$(rm "$COMMAND_LOG"); then
log 2 "error removing command log: $error"
return 1
fi
fi
}
init_command_log() {
if ! delete_command_log; then
exit 1
fi
echo "******** $(date +"%Y-%m-%d %H:%M:%S") $BATS_TEST_NAME COMMANDS ********" >> "$COMMAND_LOG"
export TEST_FILE_FOLDER
return 0
}
check_versity_vars() {
if [ -z "$LOCAL_FOLDER" ]; then
log 1 "LOCAL_FOLDER missing"
exit 1
return 1
fi
if [ ! -d "$LOCAL_FOLDER" ]; then
if ! error=$(mkdir -p "$LOCAL_FOLDER"); then
log 2 "error creating local posix folder: $error"
exit 1
return 1
fi
fi
if [ -n "$VERSIONING_DIR" ] && [ ! -d "$VERSIONING_DIR" ]; then
@@ -238,51 +248,53 @@ check_versity_vars() {
fi
if [ -z "$VERSITY_EXE" ]; then
log 1 "VERSITY_EXE missing"
exit 1
return 1
fi
if [ -z "$BACKEND" ]; then
log 1 "BACKEND missing"
exit 1
return 1
fi
export LOCAL_FOLDER VERSITY_EXE BACKEND
if [ "$BACKEND" == 's3' ]; then
if [ -z "$AWS_ACCESS_KEY_ID_TWO" ]; then
log 1 "AWS_ACCESS_KEY_ID_TWO missing"
exit 1
return 1
fi
if [ -z "$AWS_SECRET_ACCESS_KEY_TWO" ]; then
log 1 "AWS_SECRET_ACCESS_KEY_TWO missing"
exit 1
return 1
fi
export AWS_ACCESS_KEY_ID_TWO AWS_SECRET_ACCESS_KEY_TWO
fi
if [[ -r $GOCOVERDIR ]]; then
if [[ -n $GOCOVERDIR ]]; then
export GOCOVERDIR=$GOCOVERDIR
fi
if [[ $RUN_USERS == "true" ]]; then
check_user_vars
if [[ $RUN_USERS == "true" ]] && ! check_user_vars; then
log 2 "error checking versitygw users env vars"
return 1
fi
return 0
}
check_user_vars() {
if [ -z "$USERNAME_ONE" ]; then
log 1 "USERNAME_ONE missing"
exit 1
return 1
fi
if [ -z "$PASSWORD_ONE" ]; then
log 1 "PASSWORD_ONE missing"
exit 1
return 1
fi
if [ -z "$USERNAME_TWO" ]; then
log 1 "USERNAME_TWO missing"
exit 1
return 1
fi
if [ -z "$PASSWORD_TWO" ]; then
log 1 "PASSWORD_TWO missing"
exit 1
return 1
fi
if [ "$AUTOGENERATE_USERS" != "true" ] && [ "$AUTOGENERATE_USERS" != "false" ]; then
log 1 "AUTOGENERATE_USERS must be 'true' or 'false'"
@@ -303,12 +315,12 @@ check_user_vars() {
if [[ "$IAM_TYPE" == "folder" ]]; then
if [ -z "$USERS_FOLDER" ]; then
log 1 "USERS_FOLDER missing"
exit 1
return 1
fi
if [ ! -d "$USERS_FOLDER" ]; then
if ! mkdir_error=$(mkdir "$USERS_FOLDER" 2>&1); then
log 1 "error creating users folder: $mkdir_error"
exit 1
return 1
fi
fi
IAM_PARAMS="--iam-dir=$USERS_FOLDER"
@@ -318,7 +330,7 @@ check_user_vars() {
if [[ $IAM_TYPE == "s3" ]]; then
if [ -z "$USERS_BUCKET" ]; then
log 1 "error creating USERS_BUCKET"
exit 1
return 1
fi
IAM_PARAMS="--s3-iam-access $AWS_ACCESS_KEY_ID --s3-iam-secret $AWS_SECRET_ACCESS_KEY \
--s3-iam-region $AWS_REGION --s3-iam-bucket $USERS_BUCKET --s3-iam-endpoint $AWS_ENDPOINT_URL \
@@ -327,26 +339,109 @@ check_user_vars() {
return 0
fi
log 1 "unrecognized IAM_TYPE value: $IAM_TYPE"
exit 1
return 1
}
log_cleanup() {
if [ -e "$TEST_LOG_FILE.tmp" ]; then
if ! error=$(cat "$TEST_LOG_FILE.tmp" >> "$TEST_LOG_FILE" 2>&1); then
log 3 "error appending temp log to main log: $error"
fi
if ! delete_temp_log_if_exists; then
log 3 "error deleting temp log"
delete_command_log() {
if [ -f "$COMMAND_LOG" ]; then
if ! error=$(rm "$COMMAND_LOG"); then
log 2 "error removing command log: $error"
return 1
fi
fi
return 0
}
delete_temp_log_if_exists() {
if [ -e "$TEST_LOG_FILE.tmp" ]; then
if ! error=$(rm "$TEST_LOG_FILE.tmp" 2>&1); then
init_command_log() {
if ! delete_command_log; then
log 1 "error deleting old command log"
return 1
fi
if ! echo "******** $(date +"%Y-%m-%d %H:%M:%S") $BATS_TEST_NAME COMMANDS ********" >> "$COMMAND_LOG"; then
log 1 "fatal error: unable to write to file '$COMMAND_LOG'"
return 1
fi
return 0
}
main_log_cleanup() {
if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then
if ! error=$(cat "${TEST_LOG_FILE}.${TEST_ID}" >> "$TEST_LOG_FILE" 2>&1); then
log 2 "error appending temp log to main log: $error"
fi
if ! error=$(rm "${TEST_LOG_FILE}.${TEST_ID}" 2>&1); then
log 2 "error deleting temp log: $error"
return 1
fi
fi
return 0
}
teardown_logs() {
local response
if [[ $LOG_LEVEL -ge 4 ]] || [[ -n "$TIME_LOG" ]]; then
teardown_time_log
fi
if [[ -f "${TEST_LOG_FILE}.${TEST_ID}" ]]; then
echo "********************************** END TEST LOG **********************************" >> "${TEST_LOG_FILE}.${TEST_ID}"
fi
if [[ -f "$COMMAND_LOG" ]]; then
teardown_command_log
fi
if [ -f "${VERSITY_LOG_FILE}.${TEST_ID}.1" ]; then
teardown_versity_log 1
fi
if [ -f "${VERSITY_LOG_FILE}.${TEST_ID}.2" ]; then
teardown_versity_log 2
fi
if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ] && [ "$BATS_TEST_COMPLETED" != "1" ]; then
cat "${TEST_LOG_FILE}.${TEST_ID}"
fi
main_log_cleanup
}
teardown_time_log() {
local end_time total_time
end_time=$(date +%s)
total_time=$((end_time - START_TIME))
log 4 "Total test time: $total_time"
if [[ -n "$TIME_LOG" ]]; then
if ! echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"; then
log 3 "unable to write to '$TIME_LOG', check permissions"
return 1
fi
fi
return 0
}
teardown_command_log() {
echo "**********************************************************************************" >> "$COMMAND_LOG"
if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then
cat "$COMMAND_LOG" >> "${TEST_LOG_FILE}.${TEST_ID}"
elif [ "$BATS_TEST_COMPLETED" != "1" ]; then
cat "$COMMAND_LOG"
fi
if ! delete_command_log; then
log 2 "error deleting command log"
return 1
fi
return 0
}
teardown_versity_log() {
if ! check_param_count_v2 "versitygw process ID" 1 $#; then
return 1
fi
echo "**********************************************************************************" >> "${VERSITY_LOG_FILE}.${TEST_ID}.$1"
if [ -f "${TEST_LOG_FILE}.${TEST_ID}" ]; then
cat "${VERSITY_LOG_FILE}.${TEST_ID}.$1" >> "${TEST_LOG_FILE}.${TEST_ID}"
elif [ "$BATS_TEST_COMPLETED" != "1" ]; then
cat "${VERSITY_LOG_FILE}.${TEST_ID}.$1"
fi
if ! response=$(rm "${VERSITY_LOG_FILE}.${TEST_ID}.$1" 2>&1); then
log 3 "error deleting log file '${VERSITY_LOG_FILE}.${TEST_ID}.$1': $response"
fi
return 0
}
tests/logger.sh
+52 -29
View File
@@ -62,27 +62,36 @@ log_with_stack_ref() {
log_mask "$log_level" "$2" "$3"
return 0
fi
log_message "$log_level" "$2" "$3"
if ! log_message "$log_level" "$2" "$3"; then
return 1
fi
return 0
}
log_mask() {
if ! check_log_params "log_mask" "level, string, stack reference" 3 $#; then
return 1
fi
if ! mask_args "$2"; then
echo "error masking args"
local response masked_data
if ! response=$(mask_args "$2" 2>&1); then
echo "error masking args: $response"
return 1
fi
masked_data="$response"
log_message "$log_level" "$masked_data" "$3"
if ! log_message "$1" "$masked_data" "$3"; then
return 1
fi
return 0
}
mask_args() {
if ! check_log_params "mask_args" "string" 1 $#; then
return 1
fi
unmasked_array=()
masked_data=""
local unmasked_array=() first_line masked_data="" response masked_line
while IFS= read -r line; do
unmasked_array+=("$line")
done <<< "$1"
@@ -90,60 +99,69 @@ mask_args() {
# shellcheck disable=SC2068
first_line=true
for line in "${unmasked_array[@]}"; do
if ! mask_arg_array "$line"; then
echo "error masking arg array"
if ! response=$(mask_arg_array "$line" 2>&1); then
echo "error masking arg array: $response"
return 1
fi
masked_line="$response"
if [ "$first_line" == "true" ]; then
masked_data="${masked_args[*]}"
masked_data="$masked_line"
first_line="false"
else
masked_data+=$(printf "\n%s" "${masked_args[*]}")
masked_data+=$(printf "\n%s" "$masked_line")
fi
done
echo "$masked_data"
return 0
}
mask_arg_array() {
if [ $# -eq 0 ]; then
echo "'mask_arg_array' requires parameters"
echo "'mask_arg_array' requires data to be logged"
return 1
fi
mask_next=false
is_access=false
masked_args=() # Initialize an array to hold the masked arguments
local mask_next=false is_access=false masked_args=() # Initialize an array to hold the masked arguments
# shellcheck disable=SC2068
for arg in $@; do
if ! check_arg_for_mask "$arg"; then
echo "error checking arg for mask"
if ! response=$(check_arg_for_mask "$mask_next" "$is_access" "$arg" 2>&1); then
echo "error checking arg for mask: $response"
return 1
fi
read -r mask_next is_access masked_arg <<< "$response"
masked_args+=("$masked_arg")
done
echo "${masked_args[*]}"
return 0
}
check_arg_for_mask() {
if ! check_log_params "check_arg_for_mask" "arg" 1 $#; then
if ! check_log_params "check_arg_for_mask" "mask next, is access ID, arg" 3 $#; then
return 1
fi
local mask_next="$1" is_access="$2" arg="$3"
if [[ $mask_next == true ]]; then
if [ "$is_access" == "true" ]; then
masked_args+=("${arg:0:4}****")
masked_arg="${arg:0:4}****"
is_access=false
else
masked_args+=("\********")
masked_arg="\********"
fi
mask_next=false
elif [[ "$arg" == --secret_key=* ]]; then
masked_args+=("--secret_key=********")
masked_arg="--secret_key=********"
elif [[ "$arg" == --secret=* ]]; then
masked_args+=("--secret=********")
masked_arg="--secret=********"
elif [[ "$arg" == --access=* ]]; then
masked_args+=("${arg:0:13}****")
masked_arg="${arg:0:13}****"
elif [[ "$arg" == --access_key=* ]]; then
masked_args+=("${arg:0:17}****")
masked_arg="${arg:0:17}****"
elif [[ "$arg" == *"Credential="* ]]; then
masked_args+=("$(echo "$arg" | sed -E 's/(Credential=[A-Z0-9]{5})[^\/]*/\1****/g')")
masked_arg="$(echo "$arg" | sed -E 's/(Credential=[A-Z0-9]{5})[^\/]*/\1****/g')"
elif [[ "$arg" == *"AWS_ACCESS_KEY_ID="* ]]; then
masked_args+=("AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:0:5}****")
masked_arg="AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:0:5}****"
else
if [[ "$arg" == "--secret_key" ]] || [[ "$arg" == "--secret" ]] || [[ "$arg" == "--s3-iam-secret" ]]; then
mask_next=true
@@ -151,21 +169,26 @@ check_arg_for_mask() {
mask_next=true
is_access=true
fi
masked_args+=("$arg")
masked_arg="$arg"
fi
echo "$mask_next $is_access $masked_arg"
return 0
}
log_message() {
if ! check_log_params "log_message" "level, message, stack reference" 3 $#; then
return 1
fi
local bash_source_ref=$(($3+1))
local bash_source_ref=$(($3+1)) now
now="$(date "+%Y-%m-%d %H:%M:%S")"
if [[ ( "$1" == "CRIT" ) || ( "$1" == "ERROR" ) ]]; then
printf "%s\n" "$now $1 $2" >&2
fi
if [[ -n "$TEST_LOG_FILE" ]]; then
printf "%s\n" "$now ${BASH_SOURCE[$bash_source_ref]}:${BASH_LINENO[$3]} $1 $2" >> "$TEST_LOG_FILE.tmp"
if ! printf "%s\n" "$now ${BASH_SOURCE[$bash_source_ref]}:${BASH_LINENO[$3]} $1 $2" >> "${TEST_LOG_FILE}.${TEST_ID}"; then
echo "unable to write to log file '${TEST_LOG_FILE}.${TEST_ID}'"
return 1
fi
fi
sync
return 0
}
tests/report.sh
+16 -4
View File
@@ -194,6 +194,8 @@ write_to_coverage_log() {
if ! check_param_count_v2 "string" 1 $#; then
return 1
fi
local response
echo "$1" >> "$COVERAGE_LOG"
sort "$COVERAGE_LOG" | uniq > "${COVERAGE_LOG}.tmp"
mv "${COVERAGE_LOG}.tmp" "$COVERAGE_LOG"
@@ -224,14 +226,19 @@ parse_command_info() {
if ! check_param_count_v2 "command string" 1 $#; then
return 1
fi
local response command_info
if [[ "$1" == *"curl "* ]]; then
if ! command_info=$(parse_curl_rest_command "$1" 2>&1); then
echo "error parsing rest command: $command_info"
if ! response=$(parse_curl_rest_command "$1" 2>&1); then
echo "error parsing rest command: $response"
return 1
fi
command_info="$response"
else
command_info="OTHER"
fi
echo "$command_info"
return 0
}
record_command_v2() {
@@ -239,13 +246,17 @@ record_command_v2() {
log 5 "no coverage log set"
return 0
fi
local response command_info
if ! check_param_count_v2 "command string" 1 $#; then
return 1
fi
if ! parse_command_info "$1"; then
log 2 "error parsing command info"
if ! response=$(parse_command_info "$1" 2>&1); then
log 2 "error parsing command info: $response"
return 1
fi
command_info="$response"
if [ "$command_info" == "OTHER" ]; then
return 0
fi
@@ -253,4 +264,5 @@ record_command_v2() {
log 2 "error writing to coverage log"
return 1
fi
return 0
}
tests/setup.sh
+43 -37
View File
@@ -64,9 +64,24 @@ static_user_versitygw_setup() {
# bats setup function
setup() {
base_setup
TEST_ID=$(date +"%Y%m%d-%H%M%S").$(basename "$BATS_TEST_FILENAME").$BATS_TEST_NAME
export TEST_ID
setup_test_log_file
source_config_file
if ! setup_test_log_file; then
log 1 "error creating test log file"
return 1
fi
if [ -n "$TEST_LOG_FILE" ]; then
printf "\n%s\n\n" "**** $TEST_ID ****" >> "${TEST_LOG_FILE}.${TEST_ID}"
fi
if ! base_setup; then
log 2 "error checking env vars or running versitygw"
return 1
fi
if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ]; then
if ! static_user_versitygw_setup; then
@@ -77,8 +92,8 @@ setup() {
log 4 "Running test $BATS_TEST_NAME"
if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
start_time=$(date +%s)
export start_time
START_TIME=$(date +%s)
export START_TIME
fi
if [[ $RUN_S3CMD == true ]]; then
@@ -90,47 +105,40 @@ setup() {
export S3CMD_CONFIG S3CMD_OPTS
fi
if [[ $RUN_MC == true ]]; then
check_add_mc_alias
if [[ $RUN_MC == true ]] && ! check_add_mc_alias; then
log 1 "error checking for or adding mc alias"
return 1
fi
export AWS_PROFILE
log 4 "********** END SETUP **********"
}
post_versity_cleanup() {
if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
end_time=$(date +%s)
total_time=$((end_time - start_time))
log 4 "Total test time: $total_time"
if [[ -n "$TIME_LOG" ]]; then
echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
fi
fi
if [[ "$BATS_TEST_COMPLETED" -ne 1 ]]; then
if [[ -e "$COMMAND_LOG" ]]; then
cat "$COMMAND_LOG"
echo "**********************************************************************************"
fi
if [[ -e "$TEST_LOG_FILE.tmp" ]]; then
echo "********************************** LOG *******************************************"
cat "$TEST_LOG_FILE.tmp"
echo "**********************************************************************************"
fi
fi
if ! delete_command_log; then
log 3 "error deleting command log"
fi
log_cleanup
return 0
}
# bats teardown function
teardown() {
# shellcheck disable=SC2154
local response proc_status_one proc_status_two
response=$(check_versity_process_status)
read -r proc_status_one proc_status_two <<< "$response"
if [ "$proc_status_one" == "none" ] || [[ ( "$proc_status_one" == "running" ) && (( "$proc_status_two" == "none") || ( "$proc_status_two" == "running")) ]]; then
bucket_and_user_cleanup
fi
if [ "$proc_status_one" == "running" ]; then
stop_versity_process "$VERSITYGW_PID_1"
fi
if [ "$proc_status_two" == "running" ]; then
stop_versity_process "$VERSITYGW_PID_2"
fi
remove_test_file_folder_if_desired
teardown_logs
}
bucket_and_user_cleanup() {
log 4 "********** BEGIN TEARDOWN **********"
if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then
log 3 "error cleaning up buckets after test"
return 1
fi
if [ "$SKIP_USERS_TESTS" != "true" ]; then
if [ -n "$USERNAME_ONE" ]; then
@@ -147,7 +155,5 @@ teardown() {
if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then
log 3 "error deleting autocreated users"
fi
remove_test_log_file_if_desired
stop_versity
post_versity_cleanup
return 0
}
tests/setup_static.sh
+1 -1
View File
@@ -49,6 +49,6 @@ fi
# shellcheck disable=SC2034
RECREATE_BUCKETS=false
if ! stop_versity; then
if ! stop_versity_process "$VERSITYGW_PID_1"; then
log 2 "error stopping versity"
fi
tests/setup_unit.sh
+1 -2
View File
@@ -24,6 +24,5 @@ setup() {
}
teardown() {
remove_test_log_file_if_desired
log_cleanup
teardown_logs
}
tests/versity.sh
+114 -59
View File
@@ -14,53 +14,89 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/list_buckets.sh
start_versity_process() {
if ! check_param_count "start_versity_process" "versity app index" 1 $#; then
if ! check_param_count_gt "versity app index, command array" 2 $#; then
exit 1
fi
build_run_and_log_command
# shellcheck disable=SC2181
if [[ $? -ne 0 ]]; then
sleep 1
if [ -n "$VERSITY_LOG_FILE" ]; then
log 1 "error running versitygw command: $(cat "$VERSITY_LOG_FILE")"
return 1
fi
return 1
fi
eval versitygw_pid_"$1"=$!
if [ -n "$VERSITY_LOG_FILE" ]; then
process_info="Versity process $1, PID $!"
echo "$process_info" >> "$VERSITY_LOG_FILE"
fi
log 4 "$process_info"
local pid
eval pid=\$versitygw_pid_"$1"
sleep 1
local response process_id
if ! check_result=$(kill -0 "$pid" 2>&1); then
log 1 "versitygw failed to start: $check_result"
if [ -n "$VERSITY_LOG_FILE" ]; then
log 1 "log data: $(cat "$VERSITY_LOG_FILE")"
fi
if ! response=$(build_run_and_log_command "$1" "${@:2}" 2>&1); then
log 2 "error building, logging, and/or running 'versitygw' executable: $response"
return 1
fi
export versitygw_pid_"$1"
process_id="$response"
printf -v "VERSITYGW_PID_$1" '%s' "$process_id"
export VERSITYGW_PID_"$1"
return 0
}
build_run_and_log_command() {
IFS=' ' read -r -a full_command <<< "${base_command[@]}"
if ! check_param_count_gt "versitygw process number (1 or 2), command array" 2 $#; then
return 1
fi
local response full_command versitygw_log_file_name="" pid check_result
IFS=' ' read -r -a full_command <<< "${@:2}"
log 5 "versity command: ${full_command[*]}"
if [ -n "$COMMAND_LOG" ]; then
mask_args "${full_command[*]}"
if ! response=$(mask_args "${full_command[*]}" 2>&1); then
log 2 "error masking versitygw command"
return 1
fi
# shellcheck disable=SC2154
echo "${masked_args[@]}" >> "$COMMAND_LOG"
echo "$response" >> "$COMMAND_LOG"
fi
if [ -n "$VERSITY_LOG_FILE" ]; then
"${full_command[@]}" >> "$VERSITY_LOG_FILE" 2>&1 &
versitygw_log_file_name="$VERSITY_LOG_FILE.$TEST_ID".$1
echo "****************************** VERSITYGW $1 LOG ***********************************" >> "$versitygw_log_file_name"
"${full_command[@]}" >> "$versitygw_log_file_name" 2>&1 &
else
"${full_command[@]}" 2>&1 &
"${full_command[@]}" >/dev/null 2>&1 &
fi
pid="$!"
if ! verify_process_started "$pid" "$versitygw_log_file_name"; then
return 1
fi
echo "$pid"
return 0
}
verify_process_started() {
if ! check_param_count_ge_le "pid, log file (if any)" 1 2 $#; then
return 1
fi
local check_result process_running="false" proc_state
for ((check_num=1; check_num<=3; check_num++)); do
sleep 1
if [ "$process_running" == "false" ] && check_result=$(kill -0 "$1" 2>&1); then
process_running="true"
fi
if [ "$process_running" == "true" ]; then
proc_state=$(ps -p "$1" -o state= 2>/dev/null | tr -d ' ')
if [ "$proc_state" != "Z" ] && list_buckets_rest "" "" >/dev/null; then
return 0
fi
fi
done
if [ "$process_running" == "false" ]; then
log 1 "versitygw failed to start or crashed: $check_result"
elif [[ "$proc_state" == "Z" ]]; then
log 1 "versitygw process running in zombie state"
else
log 1 "process running in state '$proc_state', but not communicating properly"
fi
if [[ -n "$2" ]]; then
log 1 "log data: '$(cat "$2")'"
fi
return 1
}
run_versity_app_posix() {
@@ -84,9 +120,11 @@ run_versity_app_posix() {
base_command+=(--versioning-dir "$VERSIONING_DIR")
fi
base_command+=("$LOCAL_FOLDER")
export base_command
start_versity_process "$3"
if ! start_versity_process "$3" "${base_command[@]}"; then
log 1 "error starting versity process"
return 1
fi
return 0
}
@@ -102,9 +140,11 @@ run_versity_app_scoutfs() {
base_command+=(--port ":$PORT")
fi
base_command+=(scoutfs "$LOCAL_FOLDER")
export base_command
start_versity_process "$3"
if ! start_versity_process "$3" "${base_command[@]}"; then
log 1 "error starting versity process"
return 1
fi
return 0
}
@@ -122,20 +162,30 @@ run_versity_app_s3() {
base_command+=(--port ":7071")
fi
base_command+=(s3 --access="$AWS_ACCESS_KEY_ID_TWO" --secret="$AWS_SECRET_ACCESS_KEY_TWO" --region="$AWS_REGION" --endpoint=https://s3.amazonaws.com)
export base_command
start_versity_process "$1"
if ! start_versity_process "$1" "${base_command[@]}"; then
log 2 "error starting versity process"
return 1
fi
return 0
}
run_versity_app() {
if [[ $BACKEND == 'posix' ]]; then
run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"
if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then
return 1
fi
elif [[ $BACKEND == 'scoutfs' ]]; then
run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"
if ! run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then
return 1
fi
elif [[ $BACKEND == 's3' ]]; then
run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"
run_versity_app_s3 "2"
if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then
return 1
fi
if ! run_versity_app_s3 "2"; then
return 1
fi
else
log 1 "unrecognized backend type $BACKEND"
return 1
@@ -151,9 +201,10 @@ run_versity_app() {
teardown
return 1
fi
return 0
}
stop_single_process() {
stop_versity_process() {
if ! check_param_count "stop_single_process" "versitygw PID" 1 $#; then
return 1
fi
@@ -161,27 +212,31 @@ stop_single_process() {
# shellcheck disable=SC2086
if ps_result=$(ps -p $1 2>&1) > /dev/null; then
kill "$1"
wait "$1" || true
wait "$1" 2>/dev/null || true
else
log 3 "error stopping versity app: $ps_result"
fi
}
stop_versity() {
if [ "$RUN_VERSITYGW" == "false" ]; then
return
fi
if [[ -z "$versitygw_pid_1" ]]; then
return
fi
# shellcheck disable=SC2154
if ! stop_single_process "$versitygw_pid_1"; then
log 2 "error stopping versity process"
fi
if [[ $BACKEND == 's3' ]] && [[ -n "$versitygw_pid_2" ]]; then
# shellcheck disable=SC2154
if ! stop_single_process "$versitygw_pid_2"; then
log 2 "error stopping versity process two"
check_versity_process_status() {
local status_one="" status_two=""
status_one="none"
status_two="none"
if [ "$RUN_VERSITYGW" == "true" ]; then
if [[ -n "$VERSITYGW_PID_1" ]] && verify_process_started "$VERSITYGW_PID_1" >/dev/null; then
status_one="running"
else
status_one="failed"
fi
if [ "$BACKEND" == "s3" ]; then
if [ -n "$VERSITYGW_PID_2" ] && verify_process_started "$VERSITYGW_PID_2" >/dev/null; then
status_two="running"
else
status_two="failed"
fi
fi
fi
}
echo "$status_one $status_two"
return 0
}