test: ListObjectsV2 REST capability, initial multi-page testing

tests/.env.docker.default

@@ -25,4 +25,5 @@ USERNAME_TWO=HIJKLMN
 PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 RECREATE_BUCKETS=true
-REMOVE_TEST_FILE_FOLDER=true
+REMOVE_TEST_FILE_FOLDER=true
+VERSIONING_DIR=/tmp/versioning

tests/README.md

@@ -75,10 +75,11 @@ To communicate directly with s3, in order to compare the gateway results to dire
 1.  Copy `.secrets.default` to `.secrets` in the `tests` folder and change the parameters and add the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
 2.  By default, the dockerfile uses the **arm** architecture (usually modern Mac).  If using **amd** (usually earlier Mac or Linux), you can either replace the corresponding `ARG` values directly, or with `arg="<param>=<amd library or folder>"`  Also, you can determine which is used by your OS with `uname -a`.
 3.  Build and run the `Dockerfile_test_bats` file.  Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively, if not using the defaults.  Example:  `docker build -t <tag> -f Dockerfile_test_bats --build-arg="SECRETS_FILE=<file>" --build-arg="CONFIG_FILE=<file>" .`.
+4.  To run the entire suite, run `docker run -it <image name>`.  To run an individual suite, pass in the name of the suite as defined in `tests/run.sh` (e.g. REST tests -> `docker run -it <image name> rest`).  Also, multiple specific suites can be run, if separated by comma.
 
 ## Instructions - Running with docker-compose
 
-A file named `docker-compose-bats.yml` is provided in the root folder.  Four configurations are provided:
+A file named `docker-compose-bats.yml` is provided in the root folder.  A few configurations are provided, and you can also create your own provided you have a secrets and config file:
 * insecure (without certificates), with creation/removal of buckets
 * secure, posix backend, with static buckets
 * secure, posix backend, with creation/removal of buckets

tests/rest_scripts/list_objects.sh

@@ -20,24 +20,55 @@ source ./tests/rest_scripts/rest.sh
 
 # shellcheck disable=SC2153
 bucket_name="$BUCKET_NAME"
+version_two="${VERSION_TWO:-FALSE}"
+max_keys="${MAX_KEYS:-0}"
+# shellcheck disable=SC2153
+if [ "$CONTINUATION_TOKEN" != "" ]; then
+  continuation_token=$(jq -rn --arg token "$CONTINUATION_TOKEN" '$token | @uri')
+fi
 
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
 
 #x-amz-object-attributes:ETag
 canonical_request="GET
 /$bucket_name
+"
 
+if [ "$CONTINUATION_TOKEN" != "" ]; then
+  add_parameter "canonical_request" "continuation-token=$continuation_token"
+fi
+if [ "$version_two" != "FALSE" ]; then
+  add_parameter "canonical_request" "list-type=2"
+fi
+if [ "$max_keys" -ne 0 ]; then
+  add_parameter "canonical_request" "max-keys=$max_keys"
+fi
+first_param_added="false"
+
+canonical_request+="
 host:$host
 x-amz-content-sha256:UNSIGNED-PAYLOAD
 x-amz-date:$current_date_time
 
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
-
 create_canonical_hash_sts_and_signature
 
-curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name"
--H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+curl_command+=(curl -ks -w "\"%{http_code}\"")
+url="'$AWS_ENDPOINT_URL/$bucket_name"
+if [ "$CONTINUATION_TOKEN" != "" ]; then
+  add_parameter "url" "continuation-token=$continuation_token"
+fi
+if [ "$version_two" != "FALSE" ]; then
+  add_parameter "url" "list-type=2"
+fi
+if [ "$max_keys" -ne 0 ]; then
+  add_parameter "url" "max-keys=$max_keys"
+fi
+first_param_added="false"
+url+="'"
+curl_command+=("$url")
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
 -H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
 -H "\"x-amz-date: $current_date_time\""
 -o "$OUTPUT_FILE")

tests/rest_scripts/rest.sh

@@ -65,3 +65,18 @@ $canonical_request_hash"
   curl_command=()
   add_command_recording_if_enabled
 }
+
+add_parameter() {
+  if [ "$#" -ne 2 ]; then
+    return
+  fi
+  if [ "$first_param_added" != "true" ]; then
+    if [ "$1" == "url" ]; then
+      eval "$1"+='?'
+    fi
+    eval "$1"+="$2"
+    first_param_added="true"
+  else
+    eval "$1"+="'&$2'"
+  fi
+}

tests/test_rest.sh

@@ -43,6 +43,7 @@ source ./tests/util/util_rest.sh
 source ./tests/util/util_tags.sh
 source ./tests/util/util_time.sh
 source ./tests/util/util_versioning.sh
+source ./tests/util/util_xml.sh
 
 export RUN_USERS=true
 
@@ -437,3 +438,34 @@ export RUN_USERS=true
   run put_and_check_policy_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/policy_file.txt" "Allow" "$USERNAME_ONE" "s3:PutBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME"
   assert_success
 }
+
+@test "REST - list objects v2 - invalid continuation token" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/993"
+  fi
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test_file"
+  test_file_two="test_file_2"
+  test_file_three="test_file_3"
+  run create_test_files "$test_file" "$test_file_two" "$test_file_three"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_two" "$BUCKET_ONE_NAME" "$test_file_two"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_three" "$BUCKET_ONE_NAME" "$test_file_three"
+  assert_success
+
+  run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two"
+  assert_success
+  continuation_token=$output
+
+  # interestingly, AWS appears to accept continuation tokens that are a few characters off, so have to remove three chars
+  run list_objects_check_continuation_error "$BUCKET_ONE_NAME" "${continuation_token:0:${#continuation_token}-3}"
+  assert_success
+}

tests/util/util_list_objects.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 source ./tests/commands/list_objects_v2.sh
+source ./tests/util/util_xml.sh
 
 # Copyright 2024 Versity Software
 # This file is licensed under the Apache License, Version 2.0
@@ -242,4 +243,61 @@ list_objects_with_user_rest_verify_success() {
     return 1
   fi
   return 0
+}
+
+list_objects_check_params_get_token() {
+  if [ $# -ne 3 ]; then
+    log 2 "'list_objects_check_params_get_token' requires bucket name, files"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error attempting to get bucket ACL response: $result"
+    return 1
+  fi
+  log 5 "objects: $(cat "$TEST_FILE_FOLDER/objects.txt")"
+  if ! list_bucket_result=$(xmllint --xpath '//*[local-name()="ListBucketResult"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then
+    log 2 "error getting list bucket result: $list_bucket_result"
+    return 1
+  fi
+  if ! check_xml_element <(echo "$list_bucket_result") "$2" "Key"; then
+    log 2 "key mismatch"
+    return 1
+  fi
+  if ! check_xml_element <(echo "$list_bucket_result") "1" "MaxKeys"; then
+    log 2 "max keys mismatch"
+    return 1
+  fi
+  if ! check_xml_element <(echo "$list_bucket_result") "1" "KeyCount"; then
+    log 2 "key count mismatch"
+    return 1
+  fi
+  if ! check_xml_element <(echo "$list_bucket_result") "true" "IsTruncated"; then
+    log 2 "key count mismatch"
+    return 1
+  fi
+  if ! continuation_token=$(xmllint --xpath '//*[local-name()="NextContinuationToken"]/text()' <(echo "$list_bucket_result") 2>&1); then
+    log 2 "error getting next continuation token: $continuation_token"
+    return 1
+  fi
+  echo "$continuation_token"
+  return 0
+}
+
+list_objects_check_continuation_error() {
+  if [ $# -ne 2 ]; then
+    log 2 "'list_objects_check_continuation_error' requires bucket name, continuation token"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 CONTINUATION_TOKEN="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error attempting to get bucket ACL response: $result"
+    return 1
+  fi
+  if [ "$result" != "400" ]; then
+    log 2 "expected result code of '400' was '$result'"
+    return 1
+  fi
+  if ! check_xml_element "$TEST_FILE_FOLDER/objects.txt" "InvalidArgument" "Error" "Code"; then
+    log 2 "invalid error code"
+    return 1
+  fi
 }

tests/util/util_xml.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+check_xml_element() {
+  if [ $# -lt 3 ]; then
+    log 2 "'check_xml_element' requires data source, expected value, XML tree"
+    return 1
+  fi
+  local xpath='//'
+  for tree_val in "${@:3}"; do
+    xpath+='*[local-name()="'$tree_val'"]/'
+  done
+  xpath+='text()'
+  if ! xml_val=$(xmllint --xpath "$xpath" "$1" 2>&1); then
+    log 2 "error getting XML value matching $xpath: $xml_val"
+    return 1
+  fi
+  if [ "$2" != "$xml_val" ]; then
+    log 2 "XML data mismatch, expected '$2', actual '$xml_val'"
+    return 1
+  fi
+  return 0
+}