test: dockerfile, run script updates, README updates

tests/Dockerfile_test_bats

@@ -37,7 +37,7 @@ WORKDIR /tmp
 RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
 
 # Install mc
-RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
+RUN curl -fL https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
   --create-dirs \
   -o /usr/local/minio-binaries/mc && \
 chmod -R 755 /usr/local/minio-binaries
@@ -98,11 +98,11 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
     openssl req -new -x509 -key versitygw-docker.pem -out cert-docker.pem -days 365 \
         -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
 
-ENV WORKSPACE=.
-ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
-#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
+RUN mkdir config log && cp $CONFIG_FILE config/.env
 
-RUN mkdir log
+ENV WORKSPACE=.
+ENV VERSITYGW_TEST_ENV=/home/tester/config/.env
+#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
 
 ENTRYPOINT ["tests/run.sh"]
 CMD ["s3api,s3,s3cmd,mc,rest"]

tests/README.md

@@ -45,8 +45,13 @@
     openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
     openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
 ```
-10. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
-11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
+10. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets, for older and static bucket tests, or prefixes, for newer and non-static bucket tests.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
+11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests (not recommended), run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
+12. BATS tests can also be run directly with the format `VERSIYTGW_TEST_ENV=<env file> tests/<test file name>`, or for single tests, `VERSIYTGW_TEST_ENV=<env file> tests/<file name> -f <test name>`.  Example:  `VERSITYGW_TEST_ENV=tests/.env tests/test_rest_bucket.sh -f "REST - HeadBucket"`.
+
+#### Tags
+
+The bats tests have tag headers to allow the test user to easily find tests that check against a certain client, feature, header value, query, etc.  More info can be found in `README.md` in `tests/tags/README.md`.
 
 ### Static Bucket Mode
 
@@ -77,7 +82,11 @@ To communicate directly with s3, in order to compare the gateway results to dire
 1.  Copy `.secrets.default` to `.secrets` in the `tests` folder and change the parameters and add the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
 2.  By default, the dockerfile uses the **arm** architecture (usually modern Mac).  If using **amd** (usually earlier Mac or Linux), you can either replace the corresponding `ARG` values directly, or with `arg="<param>=<amd library or folder>"`  Also, you can determine which is used by your OS with `uname -a`.
 3.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively, if not using the defaults.  Example:  `docker build -t <tag> --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" -f tests/Dockerfile_test_bats .`.
-4.  To run the entire suite, run `docker run -it <image name>`.  This is not recommended due to the sheer amount of tests.  To run an individual suite, pass in the name of the suite as defined in `tests/run.sh` (e.g. REST tests -> `docker run -it <image name> rest`).  Also, multiple specific suites can be run, if separated by comma.
+4.  To run the entire suite, run `docker run -it <image name>`.  This is not recommended due to the sheer amount of tests.  To run an individual suite, pass in the name of the suite as defined in `tests/run.sh` (e.g. REST bucket tests -> `docker run -it <image tag or ID> rest-bucket`).  Also, multiple specific suites can be run, if separated by comma.  
+5.  To list all suites available, the `-h` tag can be passed.  Example:  `docker run -t <image tag or ID> -h`.
+6.  By default, the config is placed in the `/home/tester/config` folder inside the container.  Logs are printed to `/home/tester/log`.  To overwrite the config, an `.env` folder can be placed in a mounted host folder, and to view the logs, a mounted folder can also be used.  Example:  `docker run -v $PWD/runtime/config:/home/tester/config -v $PWD/runtime/log:/home/tester/log -t bats_test s3`
+6.  To use tag functionality, the `--tags` parameter can be passed to the container.
+7.  To troubleshoot the Docker container, use `docker run -it --entrypoint /bin/bash <image tag or ID>` to use the shell and examine the container.
 
 ## Instructions - Running with docker-compose
 

tests/docker-compose-bats.yml

@@ -34,7 +34,7 @@ services:
       args:
         - CONFIG_FILE=tests/.env.default
     image: bats_test
-    command: ["--tags","openssl"]
+    command: ["--tags","--run", "openssl"]
   direct:
     build:
       dockerfile: tests/Dockerfile_direct

tests/run.sh

@@ -21,6 +21,7 @@ show_help() {
   echo "   -h, --help                             Display this help message and exit"
   echo "                                          Separate the below by comma"
   echo "all                                       Attempt to run all tests (not recommended)"
+  echo "{suite} {pattern}                         Attempt to run tests matching pattern in single suite"
   echo_help_lines
 }
 
@@ -50,7 +51,7 @@ gather_test_files() {
 }
 
 run_set_if_matching() {
-  if ! check_param_count_v2 "set name" 1 $#; then
+  if ! check_param_count_gt "set name, test (optional)" 2 $#; then
     exit 1
   fi
   if [ "$1" == "all" ]; then
@@ -61,10 +62,18 @@ run_set_if_matching() {
     fi
     suite_run="true"
   elif [ "$run_set" == "$1" ]; then
-    echo "running '${run_sets[$idx]}' test suite ..."
-    if ! "$HOME"/bin/bats "${files[$idx]}"; then
-      echo "error running '${files[$idx]}' tests"
-      exit 1
+    if [ "$2" != "" ]; then
+      echo "running test(s) matching '$2' in '${run_sets[$idx]}' test suite ..."
+      if ! "$HOME"/bin/bats "${files[$idx]}" "-f" "$2"; then
+        echo "error running '$2' test in '${files[$idx]}' suite"
+        exit 1
+      fi
+    else
+      echo "running '${run_sets[$idx]}' test suite ..."
+      if ! "$HOME"/bin/bats "${files[$idx]}"; then
+        echo "error running '${files[$idx]}' suite"
+        exit 1
+      fi
     fi
     complete="true"
     suite_run="true"
@@ -72,14 +81,16 @@ run_set_if_matching() {
 }
 
 handle_tags() {
-  if ! check_param_count_v2 "run sets, separated by comma" 1 $#; then
-    exit 1
+  if [ "$#" -eq 0 ]; then
+    ./tests/tags/get_tests.sh "-h"
+    echo "To run from run.sh, replace 'get_tests.sh' path with '--tags'"
+    return 1
   fi
-  ./tests/tags/get_tests.sh --run "$1"
+  ./tests/tags/get_tests.sh "$@"
 }
 
 handle_param() {
-  if ! check_param_count_v2 "run sets, separated by comma" 1 $#; then
+  if ! check_param_count_gt "run sets, separated by comma, or single run set then test name" 1 $#; then
     exit 1
   fi
 
@@ -89,7 +100,7 @@ handle_param() {
   complete="false"
   suite_run="false"
   for run_set in "${run_sets[@]}"; do
-    run_set_if_matching "$1"
+    run_set_if_matching "$1" "$2"
     if [ "$complete" == "true" ]; then
       break
     fi
@@ -115,8 +126,12 @@ if [ "$1" == "--tags" ]; then
 fi
 
 IFS=',' read -ra options <<< "$1"
+if [ "$2" != "" ] && [ "${#options[@]}" -gt 1 ]; then
+  echo "cannot call multiple suites with test name"
+  exit 1
+fi
 for option in "${options[@]}"; do
-  handle_param "$option"
+  handle_param "$option" "$2"
 done
 
 # shellcheck disable=SC2086

tests/tags/README.md

@@ -0,0 +1,33 @@
+# Tag Program Examples
+
+To find specific tests, based on function, header, etc., tags are included in the test suite.  Run these examples from the root repo folder
+
+List all tag descriptions:
+```
+./tests/tags/get_tests.sh --list-tags
+```
+
+List description for a specific tag:
+```
+./test/tags/get_tests.sh --list-tags <tag name>
+```
+
+List all tests matching a tag:
+```
+./tests/tags/get_tests.sh --list <tag name>
+```
+
+List all tests matching all of the tags:
+```
+./tests/tags/get_tests.sh --list <tag name>,<tag name>
+```
+
+List all tests matching any of the tags:
+```
+./tests/tags/get_tests.sh --any <tag name>,<tag name>
+```
+
+Run all tests matching a tag:
+```
+VERSITYGW_TEST_ENV=tests/<env file> ./tests/tags/get_tests.sh --run <tag name>
+```

tests/tags/get_tests.sh

@@ -32,7 +32,7 @@ Options:
   -h, --help    Show this help.
 
 Notes:
-  - Requires VERSITYGW_TEST_ENV to be set.
+  - Running tests requires VERSITYGW_TEST_ENV to be set.
   - Tags are passed as a single comma-separated argument.
 
 Examples:
@@ -125,7 +125,10 @@ for arg in "$@"; do
   case "$arg" in
     --count) count_flag=1 ;;
     --list) list_flag=1 ;;
-    --any) tag_matching_mode="any" ;;
+    --any)
+      tag_matching_mode="any"
+      list_flag=1
+      ;;
     -h|--help) usage; exit 0 ;;
     --list-tags) list_tags_flag=1
       shift