test: parallel docker testing, some misc cleanup/fixes

This commit is contained in:
Luke McCrone
2026-08-10 18:15:27 -03:00
parent 8bd73a45ee
commit 5e284e8a87
13 changed files with 337 additions and 65 deletions
+2
View File
@@ -54,6 +54,7 @@ ENV HOME=/home/tester
RUN git clone https://github.com/bats-core/bats-core.git && \
cd bats-core && \
./install.sh /home/tester
ENV PATH="/home/tester/bin:${PATH}"
# Create a shared venv & install Python deps there
ENV VENV=/opt/venv
@@ -72,6 +73,7 @@ RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/test
RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
WORKDIR /home/tester
RUN go mod download
RUN make
RUN . $SECRETS_FILE && \
+15 -1
View File
@@ -46,7 +46,7 @@
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
```
10. The program uses two environment values for the bucket names (or prefixes): `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME`. Originally, and for static and some older tests, these are the full bucket names. In later, non-static tests, these are prefixes, and bucket suffixes are auto-generated. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets, for older and static bucket tests, or prefixes, for newer and non-static bucket tests. If you want static buckets, i.e. you don't want them to be re-created each test, set `RECREATE_BUCKETS` to `false`.
11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`. To print options, run `tests/run.sh -h`. To run all tests (not currently recommended due to long running time), run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`. To print options, run `tests/run.sh -h`. To run all tests, technically, the user can run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`, but this is not recommended due to long running time. Instead, see the docker section below to run tests in parallel.
12. BATS tests can also be run directly with the format `VERSIYTGW_TEST_ENV=<env file> tests/<test file name>`, or for single tests, `VERSIYTGW_TEST_ENV=<env file> tests/<file name> -f <test name>`. Example: `VERSITYGW_TEST_ENV=tests/.env tests/test_rest_bucket.sh -f "REST - HeadBucket"`.
#### Tags
@@ -88,6 +88,10 @@ To communicate directly with s3, in order to compare the gateway results to dire
6. To use tag functionality, the `--tags` parameter can be passed to the container.
7. To troubleshoot the Docker container, use `docker run -it --entrypoint /bin/bash <image tag or ID>` to use the shell and examine the container.
### In Parallel
The script `run_parallel.sh` is provided to allow users to more quickly run tests in parallel on the local machine. To use this script, first build and tag a docker container with the desired configuration. Next, run the script with this tag, the desired tests (can find with `./tests/run.sh -l`) separated by comma, the maximum number of parallel jobs (default 4), and the folder to place the logs.
## Instructions - Running with docker-compose
A file named `docker-compose-bats.yml` is provided in the root folder. A few configurations are provided, and you can also create your own provided you have a secrets and config file:
@@ -216,6 +220,16 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
**BYPASS_ENV_FILE**: skip loading `.env` file on startup, default is **false**
**DIRECT_CLOUDFRONT_TAG**: if communicating directly with S3, name of tag used for HTTPS cloudfront distributions for bucket websites
**GO_COMMAND_GENERATOR_EXECUTABLE**: location to set executable for go REST command generation. If not used, program will run `go run ...` each time rather than compiling. However, if using, user must ensure that executable is removed and recompiled on go code updates.
**QUICK_COMPARE_SIZE**: for some comparisons between local and endpoint files to verify correct transfer, use checksum rather than full download to compare if file or part is above this size. If not used, then just download and compare.
**WEBSITE**: website port, if using versitygw website functionality
**WEBSITE_ENDPOINT**: website endpoint, if using versitygw website functionality
## REST Scripts
REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder. To call a script, the following parameters are needed:
+7 -4
View File
@@ -224,13 +224,16 @@ search_for_first_policy_line_or_full_policy() {
}
get_bucket_policy_mc() {
if ! check_param_count "get_bucket_policy_mc" "bucket" 1 $#; then
if ! check_param_count_v2 "bucket" 1 $#; then
return 1
fi
bucket_policy=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$1" 2>&1) || get_result=$?
if [[ $get_result -ne 0 ]]; then
log 2 "error getting policy: $bucket_policy"
local bucket="$1"
local response
if ! response=$(send_command mc --insecure anonymous get-json "$MC_ALIAS/$bucket" 2>&1); then
log 2 "error getting policy: $response"
return 1
fi
printf '%s\n' "$response"
return 0
}
+1 -1
View File
@@ -276,7 +276,7 @@ delete_cloudfront_distribution() {
if [ -z "$distribution_id" ] || [ "$distribution_id" == "null" ]; then
return 0
fi
if ! response="$(env AWS_IGNORE_CONFIGURED_ENDPOINT_URLS=true aws cloudfront get-distribution-config --id "$distribution_id" 2>&1)"; then
if ! response="$(send_command env AWS_IGNORE_CONFIGURED_ENDPOINT_URLS=true aws cloudfront get-distribution-config --id "$distribution_id" 2>&1)"; then
if [[ "$response" == *"The specified distribution does not exist"* ]]; then
return 0
fi
+12 -6
View File
@@ -18,30 +18,36 @@ list_check_objects_common() {
if ! check_param_count "list_check_objects_common" "client, bucket, object one, object two" 4 $#; then
return 1
fi
if ! response=$(list_objects "$1" "$2" 2>&1); then
local client="$1" bucket="$2" object_one="$3" object_two="$4"
local response object_list object_data
if ! response=$(list_objects "$client" "$bucket" 2>&1); then
log 2 "error listing objects: $response"
return 1
fi
log 5 "response: $response"
mapfile -t object_list <<< "$response"
log 5 "object list size: ${#object_list[@]}"
local object_one_found=false
local object_two_found=false
for object_data in "${object_list[@]}"; do
if [ "$1" == "s3cmd" ]; then
object="$(echo -n "$object_data" | awk '{ for (i=4; i<=NF; i++) printf "%s%s", $i, (i<NF ? OFS : ORS) }')"
if [ "$client" == "s3cmd" ]; then
object="$(awk '{ for (i=4; i<=NF; i++) printf "%s%s", $i, (i<NF ? OFS : ORS) }' <<< "$object_data")"
elif [ "$client" == "mc" ]; then
object="$(awk '{print $6}' <<< "$object_data")"
else
object="$object_data"
fi
if [ "$object" == "$3" ] || [ "$object" == "s3://$2/$3" ]; then
if [ "$object" == "$object_one" ] || [ "$object" == "s3://$bucket/$object_one" ]; then
object_one_found=true
elif [ "$object" == "$4" ] || [ "$object" == "s3://$2/$4" ]; then
elif [ "$object" == "$object_two" ] || [ "$object" == "s3://$bucket/$object_two" ]; then
object_two_found=true
fi
done
if [ $object_one_found != true ] || [ $object_two_found != true ]; then
log 2 "$3 and/or $4 not listed (all objects: ${object_list[*]})"
log 2 "'$object_one' and/or '$object_two' not listed (all objects: ${object_list[*]})"
return 1
fi
return 0
+34 -1
View File
@@ -36,8 +36,41 @@ base_setup() {
return 0
}
get_log_name_from_placeholders() {
local test_log_file_string="$TEST_LOG_FILE_PATTERN" timestamp test_filename filename_root
if [[ "$test_log_file_string" == *"{timestamp}"* ]]; then
timestamp="$(date '+%Y%m%dT%H%M%S')"
test_log_file_string="${test_log_file_string//\{timestamp\}/$timestamp}"
fi
if [[ "$test_log_file_string" == *"{filename}"* ]]; then
test_filename="$(basename "$BATS_TEST_FILENAME")"
filename_root="${test_filename%.*}"
test_log_file_string="${test_log_file_string//\{filename\}/$filename_root}"
fi
export TEST_LOG_FILE="$test_log_file_string"
}
get_log_name() {
if [ -n "$TEST_LOG_FILE_PATTERN" ]; then
if [ -n "$TEST_LOG_FILE" ]; then
echo "both TEST_LOG_FILE_PATTERN and TEST_LOG_FILE cannot be defined"
return 1
fi
if [[ ( "$TEST_LOG_FILE_PATTERN" == *'{'* ) || ( "$TEST_LOG_FILE_PATTERN" == *'}'* ) ]]; then
get_log_name_from_placeholders
else
export TEST_LOG_FILE="$TEST_LOG_FILE_PATTERN"
fi
fi
}
setup_test_log_file() {
if [ -n "$TEST_LOG_FILE" ]; then
if [ -n "$TEST_LOG_FILE" ] || [ -n "$TEST_LOG_FILE_PATTERN" ]; then
if ! get_log_name; then
log 1 "error getting log file name"
return 1
fi
if ! error=$(touch "$TEST_LOG_FILE.$TEST_ID" 2>&1); then
log 1 "error creating test log file: $error"
return 1
+118 -40
View File
@@ -19,14 +19,37 @@ source ./tests/drivers/params.sh
show_help() {
echo "Usage: $0 [option...]"
echo " -h, --help Display this help message and exit"
echo " Separate the below by comma"
echo "all Attempt to run all tests (not recommended)"
echo " -l Only list suites (don't run)"
echo " env BATS=<path> Use a specific bats executable path"
echo " env VERSITYGW_TEST_ENV=<path> env file to pass to tests"
echo "all Attempt to run all tests (not recommended here due to time, use run_parallel.sh to run in parallel)"
echo "{suite} {pattern} Attempt to run tests matching pattern in single suite"
echo " Suites below (to run multiple, separate by comma)"
echo_help_lines
}
get_bats_executable() {
if [ -n "$BATS" ]; then
if [ ! -x "$BATS" ]; then
echo "BATS executable '$BATS' not found or not executable" >&2
return 1
fi
printf '%s\n' "$BATS"
return 0
fi
if bats_cmd="$(command -v bats 2>/dev/null)" && [[ -x "$bats_cmd" ]]; then
printf '%s\n' "$bats_cmd"
return 0
fi
echo "unable to find bats executable; set BATS=<path>" >&2
return 1
}
echo_help_lines() {
gather_test_files
local run_sets_list run_sets=() run_set description spaces_needed
run_sets_list=$(get_run_sets_and_files_if_needed)
read -r -a run_sets <<< "$run_sets_list"
for run_set in "${run_sets[@]}"; do
description=${run_set/-/ }
@@ -38,46 +61,71 @@ echo_help_lines() {
done
}
gather_test_files() {
get_run_sets_and_files_if_needed() {
if ! check_param_count_le "'true', if files desired" 1 $#; then
return 1
fi
local want_files="$1"
if [ -n "$want_files" ] && [ "$want_files" != "true" ]; then
echo "param for printing files as well must be 'true' or unset"
return 1
fi
local f files=() file_without_header file_without_sh run_set run_sets=()
while IFS= read -r f; do
if grep -q '@test' "$f"; then
files+=("$f")
if [ "$want_files" == "true" ]; then
files+=("$f")
fi
file_without_header=${f/tests\/test_/}
file_without_sh=${file_without_header/.sh/}
run_set=${file_without_sh//_/-}
run_sets+=("$run_set")
fi
done < <(find tests -name 'test_*.sh' | sort)
printf '%s\n' "${run_sets[*]}"
if [ "$want_files" == "true" ]; then
printf '%s\n' "${files[*]}"
fi
}
# return 0 for complete, 1 for continue checking suites, 2 for error
run_set_if_matching() {
if ! check_param_count_gt "set name, test (optional)" 2 $#; then
exit 1
if ! check_param_count_gt "desired run set, current run set name, current file name, test pattern (optional)" 3 $#; then
return 2
fi
if [ "$1" == "all" ]; then
echo "running '${run_sets[$idx]}' test suite ..."
if ! "$HOME"/bin/bats "${files[$idx]}"; then
echo "error running '${files[$idx]}' tests"
exit 1
local desired_run_set="$1" current_run_set="$2" current_file="$3" test_pattern="$4" bats_executable
if ! bats_executable=$(get_bats_executable); then
return 2
fi
if [ "$desired_run_set" == "all" ]; then
echo "running '$current_run_set' test suite ..."
if ! "$bats_executable" "$current_file"; then
echo "error running '$current_run_set' tests" >&2
return 2
fi
suite_run="true"
elif [ "$run_set" == "$1" ]; then
if [ "$2" != "" ]; then
echo "running test(s) matching '$2' in '${run_sets[$idx]}' test suite ..."
if ! "$HOME"/bin/bats "${files[$idx]}" "-f" "$2"; then
echo "error running '$2' test in '${files[$idx]}' suite"
exit 1
return 1
elif [ "$current_run_set" == "$desired_run_set" ]; then
if [ "$test_pattern" != "" ]; then
echo "running test(s) matching '$test_pattern' in '$current_run_set' test suite ..."
if ! "$bats_executable" "$current_file" "-f" "$test_pattern"; then
echo "error running '$test_pattern' test(s) in '$current_run_set' suite" >&2
return 2
fi
else
echo "running '${run_sets[$idx]}' test suite ..."
if ! "$HOME"/bin/bats "${files[$idx]}"; then
echo "error running '${files[$idx]}' suite"
exit 1
echo "running '$current_run_set' test suite ..."
if ! "$bats_executable" "$current_file"; then
echo "error running '$current_run_set' suite" >&2
return 2
fi
fi
complete="true"
suite_run="true"
return 0
fi
return 1
}
handle_tags() {
@@ -90,27 +138,50 @@ handle_tags() {
}
handle_param() {
if ! check_param_count_gt "run sets, separated by comma, or single run set then test name" 1 $#; then
exit 1
if ! check_param_count_ge_le "run set, test pattern (optional)" 1 2 $#; then
return 1
fi
local run_set="$1" test_pattern="$2"
local run_sets_and_files lines run_sets files idx run_result
gather_test_files
run_sets_and_files=$(get_run_sets_and_files_if_needed "true")
mapfile -t lines <<< "$run_sets_and_files"
read -r -a run_sets <<< "${lines[0]}"
read -r -a files <<< "${lines[1]}"
idx=0
complete="false"
suite_run="false"
for run_set in "${run_sets[@]}"; do
run_set_if_matching "$1" "$2"
if [ "$complete" == "true" ]; then
for idx in "${!run_sets[@]}"; do
run_result=0
run_set_if_matching "$run_set" "${run_sets[$idx]}" "${files[$idx]}" "$test_pattern" || run_result=$?
if [ "$run_result" -eq 0 ]; then
break
elif [ "$run_result" -eq 2 ]; then
echo "error running set '$run_set" >&2
return 1
fi
((idx++))
done
if [ "$suite_run" == "false" ]; then
echo "no suites matching '$1'"
exit 1
if [ "$run_result" -eq 1 ] && [ "$run_set" != "all" ]; then
echo "no suites matching '$run_set'" > /dev/stderr
return 1
fi
return 0
}
list_all_test_suites() {
local run_set_list run_sets set_length matching_test_string=""
run_set_list=$(get_run_sets_and_files_if_needed)
read -r -a run_sets <<< "$run_set_list"
set_length=${#run_sets[@]}
if [ "$set_length" -le 0 ]; then
printf '\n'
return 0
fi
matching_test_string=$(IFS=","; echo "${run_sets[*]}")
printf '%s\n' "$matching_test_string"
return 0
}
if [ $# -le 0 ] || [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
@@ -125,13 +196,20 @@ if [ "$1" == "--tags" ]; then
exit 0
fi
if [ "$1" == "-l" ]; then
list_all_test_suites
exit 0
fi
IFS=',' read -ra options <<< "$1"
if [ "$2" != "" ] && [ "${#options[@]}" -gt 1 ]; then
echo "cannot call multiple suites with test name"
echo "cannot call multiple suites with test name" >&2
exit 1
fi
for option in "${options[@]}"; do
handle_param "$option" "$2"
if ! handle_param "$option" "$2"; then
exit 1
fi
done
# shellcheck disable=SC2086
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env bash
# Copyright 2026 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/drivers/params.sh
DEFAULT_MAX_PARALLEL_JOBS=4
DEFAULT_DOCKER_LOG_FOLDER="$PWD/runtime/log"
check_for_finished_processes() {
if ! check_param_count_v2 "pid array ref, suite array ref, time array ref" 3 $#; then
return 1
fi
local -n pids_ref="$1" suites_ref="$2" times_ref="$3"
local -a pid_snapshot
local pid status run_end_time
pid_snapshot=("${!pids_ref[@]}")
for pid in "${pid_snapshot[@]}"; do
if ! kill -0 "$pid" 2>/dev/null; then
wait "$pid"
status=$?
run_end_time=$(date +%s)
printf '%s\n' "'$pid' (${suites_ref[$pid]}) finished with status '$status' (duration: $((run_end_time-${times_ref[$pid]}))s)"
unset "pids_ref[$pid]"
unset "suites_ref[$pid]"
unset "times_ref[$pid]"
fi
done
}
run_tests() {
if ! check_param_count_v2 "image tag, test list, max parallel jobs, docker log folder" 4 $#; then
return 1
fi
local image_tag="$1" test_list="$2" max_parallel_jobs="$3" docker_log_folder="$4"
local test_array pids suites times timestamp test_suite end_time duration
IFS=, read -r -a test_array <<< "$test_list"
declare -A pids suites times
timestamp="$(date '+%Y%m%dT%H%M%S')"
for test_suite in "${test_array[@]}"; do
while [ ${#pids[@]} -ge "$max_parallel_jobs" ]; do
sleep 1
check_for_finished_processes "pids" "suites" "times"
done
docker run -v "$PWD/runtime/config:/home/tester/config" -v "$docker_log_folder:/home/tester/log" -t "$image_tag" "$test_suite" > "$docker_log_folder/${test_suite}-${timestamp}.log" &
pid=$!
printf '%s\n' "'$test_suite' started (pid: '$pid')"
pids[$pid]=$pid
# shellcheck disable=SC2034
suites[$pid]=$test_suite
# shellcheck disable=SC2034
times[$pid]=$(date +%s)
done
while [ ${#pids[@]} -gt 0 ]; do
check_for_finished_processes "pids" "suites" "times"
sleep 1
done
}
if ! check_param_count_ge_le "docker image tag, test suites (separated by comma), max parallel jobs at once (default '$DEFAULT_MAX_PARALLEL_JOBS', log folder (default '$DEFAULT_DOCKER_LOG_FOLDER')" 2 4 $#; then
exit 1
fi
start_time=$(date +%s)
image_tag="$1"
test_list="$2"
if [ "$3" == "" ]; then
max_parallel_jobs="$DEFAULT_MAX_PARALLEL_JOBS"
else
if [ "$3" -le 0 ]; then
echo "max parallel jobs must be at least 1"
exit 1
fi
max_parallel_jobs="$3"
fi
if [ "$4" == "" ]; then
docker_log_folder="$DEFAULT_DOCKER_LOG_FOLDER"
else
docker_log_folder="$4"
fi
if ! docker image inspect "$image_tag" >/dev/null 2>&1; then
echo "image tagged '$image_tag' doesn't exist"
exit 1
fi
if ! mkdir -p "$docker_log_folder"; then
echo "unable to create docker log folder '$docker_log_folder'"
exit 1
fi
run_tests "$image_tag" "$test_list" "$max_parallel_jobs" "$docker_log_folder"
end_time=$(date +%s)
duration=$((end_time-start_time))
printf '%s\n' "duration: ${duration}s"
+28 -12
View File
@@ -83,11 +83,17 @@ setup() {
return 1
fi
if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ]; then
if ! static_user_versitygw_setup; then
log 2 "error setting up static versitygw users"
if [ "$RUN_USERS" == "true" ] && [ "$SKIP_USERS_TESTS" != "true" ]; then
if ! static_user_v1_cleanup; then
log 2 "error cleaning up v1 static users"
return 1
fi
if [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ] && [ "$AUTOGENERATE_USERS" == "false" ]; then
if ! static_user_versitygw_setup; then
log 2 "error setting up static versitygw users"
return 1
fi
fi
fi
log 4 "Running test $BATS_TEST_NAME"
@@ -136,21 +142,31 @@ teardown() {
teardown_logs
}
static_user_v1_cleanup() {
if [ -n "$USERNAME_ONE" ]; then
if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then
log 2 "error deleting user '$USERNAME_ONE'"
return 1
fi
fi
if [ -n "$USERNAME_TWO" ]; then
if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then
log 2 "error deleting user '$USERNAME_TWO'"
return 1
fi
fi
return 0
}
bucket_and_user_cleanup() {
log 4 "********** BEGIN TEARDOWN **********"
if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ] && ! cleanup_buckets; then
log 3 "error cleaning up buckets after test"
fi
if [ "$SKIP_USERS_TESTS" != "true" ]; then
if [ -n "$USERNAME_ONE" ]; then
if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then
log 3 "error deleting user '$USERNAME_ONE'"
fi
fi
if [ -n "$USERNAME_TWO" ]; then
if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then
log 3 "error deleting user '$USERNAME_TWO'"
fi
if ! static_user_v1_cleanup; then
log 3 "error cleaning up v1 static users"
return 1
fi
fi
if [ "$AUTOGENERATE_USERS" == "true" ] && ! delete_autogenerated_users; then
+1
View File
@@ -21,6 +21,7 @@ source ./tests/commands/command.sh
source ./tests/logger.sh
source ./tests/setup_unit.sh
# tags: unit
@test "check_for_and_or_build_go_executable" {
export GO_COMMAND_GENERATOR_EXECUTABLE=
run check_for_and_or_build_go_executable
+3
View File
@@ -28,6 +28,9 @@ source ./tests/drivers/get_bucket_location/get_bucket_location.sh
source ./tests/drivers/get_bucket_tagging/get_bucket_tagging.sh
source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh
source ./tests/drivers/get_object_tagging/get_object_tagging.sh
source ./tests/drivers/head_bucket/head_bucket_mc.sh
source ./tests/drivers/list_buckets/list_buckets.sh
source ./tests/drivers/list_objects/list_objects.sh
source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
export RUN_MC=true
+1
View File
@@ -17,6 +17,7 @@
load ./bats-support/load
load ./bats-assert/load
source ./tests/drivers/list_objects/list_objects.sh
source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
source ./tests/test_common.sh
+1
View File
@@ -160,6 +160,7 @@ source ./tests/setup.sh
assert_success
}
# tags: curl, DeleteObjects
@test "REST - DeleteObjects - quiet mode" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/2124"