diff --git a/tests/.env.docker.default b/tests/.env.docker.default
index 0ea1637..2d4187a 100644
--- a/tests/.env.docker.default
+++ b/tests/.env.docker.default
@@ -25,4 +25,5 @@ USERNAME_TWO=HIJKLMN
PASSWORD_TWO=OPQRSTU
TEST_FILE_FOLDER=$PWD/versity-gwtest-files
RECREATE_BUCKETS=true
-REMOVE_TEST_FILE_FOLDER=true
\ No newline at end of file
+REMOVE_TEST_FILE_FOLDER=true
+VERSIONING_DIR=/tmp/versioning
\ No newline at end of file
diff --git a/tests/README.md b/tests/README.md
index 8c784d9..95bdfaa 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -75,10 +75,11 @@ To communicate directly with s3, in order to compare the gateway results to dire
1. Copy `.secrets.default` to `.secrets` in the `tests` folder and change the parameters and add the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend.
2. By default, the dockerfile uses the **arm** architecture (usually modern Mac). If using **amd** (usually earlier Mac or Linux), you can either replace the corresponding `ARG` values directly, or with `arg="="` Also, you can determine which is used by your OS with `uname -a`.
3. Build and run the `Dockerfile_test_bats` file. Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively, if not using the defaults. Example: `docker build -t -f Dockerfile_test_bats --build-arg="SECRETS_FILE=" --build-arg="CONFIG_FILE=" .`.
+4. To run the entire suite, run `docker run -it `. To run an individual suite, pass in the name of the suite as defined in `tests/run.sh` (e.g. REST tests -> `docker run -it rest`). Also, multiple specific suites can be run, if separated by comma.
## Instructions - Running with docker-compose
-A file named `docker-compose-bats.yml` is provided in the root folder. Four configurations are provided:
+A file named `docker-compose-bats.yml` is provided in the root folder. A few configurations are provided, and you can also create your own provided you have a secrets and config file:
* insecure (without certificates), with creation/removal of buckets
* secure, posix backend, with static buckets
* secure, posix backend, with creation/removal of buckets
diff --git a/tests/rest_scripts/list_objects.sh b/tests/rest_scripts/list_objects.sh
index d39743e..39ef7bc 100755
--- a/tests/rest_scripts/list_objects.sh
+++ b/tests/rest_scripts/list_objects.sh
@@ -20,24 +20,55 @@ source ./tests/rest_scripts/rest.sh
# shellcheck disable=SC2153
bucket_name="$BUCKET_NAME"
+version_two="${VERSION_TWO:-FALSE}"
+max_keys="${MAX_KEYS:-0}"
+# shellcheck disable=SC2153
+if [ "$CONTINUATION_TOKEN" != "" ]; then
+ continuation_token=$(jq -rn --arg token "$CONTINUATION_TOKEN" '$token | @uri')
+fi
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
#x-amz-object-attributes:ETag
canonical_request="GET
/$bucket_name
+"
+if [ "$CONTINUATION_TOKEN" != "" ]; then
+ add_parameter "canonical_request" "continuation-token=$continuation_token"
+fi
+if [ "$version_two" != "FALSE" ]; then
+ add_parameter "canonical_request" "list-type=2"
+fi
+if [ "$max_keys" -ne 0 ]; then
+ add_parameter "canonical_request" "max-keys=$max_keys"
+fi
+first_param_added="false"
+
+canonical_request+="
host:$host
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD"
-
create_canonical_hash_sts_and_signature
-curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name"
--H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+curl_command+=(curl -ks -w "\"%{http_code}\"")
+url="'$AWS_ENDPOINT_URL/$bucket_name"
+if [ "$CONTINUATION_TOKEN" != "" ]; then
+ add_parameter "url" "continuation-token=$continuation_token"
+fi
+if [ "$version_two" != "FALSE" ]; then
+ add_parameter "url" "list-type=2"
+fi
+if [ "$max_keys" -ne 0 ]; then
+ add_parameter "url" "max-keys=$max_keys"
+fi
+first_param_added="false"
+url+="'"
+curl_command+=("$url")
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
-H "\"x-amz-date: $current_date_time\""
-o "$OUTPUT_FILE")
diff --git a/tests/rest_scripts/rest.sh b/tests/rest_scripts/rest.sh
index eb20ea0..332de21 100644
--- a/tests/rest_scripts/rest.sh
+++ b/tests/rest_scripts/rest.sh
@@ -65,3 +65,18 @@ $canonical_request_hash"
curl_command=()
add_command_recording_if_enabled
}
+
+add_parameter() {
+ if [ "$#" -ne 2 ]; then
+ return
+ fi
+ if [ "$first_param_added" != "true" ]; then
+ if [ "$1" == "url" ]; then
+ eval "$1"+='?'
+ fi
+ eval "$1"+="$2"
+ first_param_added="true"
+ else
+ eval "$1"+="'&$2'"
+ fi
+}
diff --git a/tests/test_rest.sh b/tests/test_rest.sh
index a7a2b64..5cba21b 100755
--- a/tests/test_rest.sh
+++ b/tests/test_rest.sh
@@ -43,6 +43,7 @@ source ./tests/util/util_rest.sh
source ./tests/util/util_tags.sh
source ./tests/util/util_time.sh
source ./tests/util/util_versioning.sh
+source ./tests/util/util_xml.sh
export RUN_USERS=true
@@ -437,3 +438,34 @@ export RUN_USERS=true
run put_and_check_policy_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/policy_file.txt" "Allow" "$USERNAME_ONE" "s3:PutBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME"
assert_success
}
+
+@test "REST - list objects v2 - invalid continuation token" {
+ if [ "$DIRECT" != "true" ]; then
+ skip "https://github.com/versity/versitygw/issues/993"
+ fi
+ run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+ assert_success
+
+ test_file="test_file"
+ test_file_two="test_file_2"
+ test_file_three="test_file_3"
+ run create_test_files "$test_file" "$test_file_two" "$test_file_three"
+ assert_success
+
+ run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+ assert_success
+
+ run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_two" "$BUCKET_ONE_NAME" "$test_file_two"
+ assert_success
+
+ run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_three" "$BUCKET_ONE_NAME" "$test_file_three"
+ assert_success
+
+ run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two"
+ assert_success
+ continuation_token=$output
+
+ # interestingly, AWS appears to accept continuation tokens that are a few characters off, so have to remove three chars
+ run list_objects_check_continuation_error "$BUCKET_ONE_NAME" "${continuation_token:0:${#continuation_token}-3}"
+ assert_success
+}
diff --git a/tests/util/util_list_objects.sh b/tests/util/util_list_objects.sh
index 08cb7b0..837311a 100644
--- a/tests/util/util_list_objects.sh
+++ b/tests/util/util_list_objects.sh
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
source ./tests/commands/list_objects_v2.sh
+source ./tests/util/util_xml.sh
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
@@ -242,4 +243,61 @@ list_objects_with_user_rest_verify_success() {
return 1
fi
return 0
+}
+
+list_objects_check_params_get_token() {
+ if [ $# -ne 3 ]; then
+ log 2 "'list_objects_check_params_get_token' requires bucket name, files"
+ return 1
+ fi
+ if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+ log 2 "error attempting to get bucket ACL response: $result"
+ return 1
+ fi
+ log 5 "objects: $(cat "$TEST_FILE_FOLDER/objects.txt")"
+ if ! list_bucket_result=$(xmllint --xpath '//*[local-name()="ListBucketResult"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then
+ log 2 "error getting list bucket result: $list_bucket_result"
+ return 1
+ fi
+ if ! check_xml_element <(echo "$list_bucket_result") "$2" "Key"; then
+ log 2 "key mismatch"
+ return 1
+ fi
+ if ! check_xml_element <(echo "$list_bucket_result") "1" "MaxKeys"; then
+ log 2 "max keys mismatch"
+ return 1
+ fi
+ if ! check_xml_element <(echo "$list_bucket_result") "1" "KeyCount"; then
+ log 2 "key count mismatch"
+ return 1
+ fi
+ if ! check_xml_element <(echo "$list_bucket_result") "true" "IsTruncated"; then
+ log 2 "key count mismatch"
+ return 1
+ fi
+ if ! continuation_token=$(xmllint --xpath '//*[local-name()="NextContinuationToken"]/text()' <(echo "$list_bucket_result") 2>&1); then
+ log 2 "error getting next continuation token: $continuation_token"
+ return 1
+ fi
+ echo "$continuation_token"
+ return 0
+}
+
+list_objects_check_continuation_error() {
+ if [ $# -ne 2 ]; then
+ log 2 "'list_objects_check_continuation_error' requires bucket name, continuation token"
+ return 1
+ fi
+ if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 CONTINUATION_TOKEN="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+ log 2 "error attempting to get bucket ACL response: $result"
+ return 1
+ fi
+ if [ "$result" != "400" ]; then
+ log 2 "expected result code of '400' was '$result'"
+ return 1
+ fi
+ if ! check_xml_element "$TEST_FILE_FOLDER/objects.txt" "InvalidArgument" "Error" "Code"; then
+ log 2 "invalid error code"
+ return 1
+ fi
}
\ No newline at end of file
diff --git a/tests/util/util_xml.sh b/tests/util/util_xml.sh
new file mode 100644
index 0000000..1cad8c2
--- /dev/null
+++ b/tests/util/util_xml.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+check_xml_element() {
+ if [ $# -lt 3 ]; then
+ log 2 "'check_xml_element' requires data source, expected value, XML tree"
+ return 1
+ fi
+ local xpath='//'
+ for tree_val in "${@:3}"; do
+ xpath+='*[local-name()="'$tree_val'"]/'
+ done
+ xpath+='text()'
+ if ! xml_val=$(xmllint --xpath "$xpath" "$1" 2>&1); then
+ log 2 "error getting XML value matching $xpath: $xml_val"
+ return 1
+ fi
+ if [ "$2" != "$xml_val" ]; then
+ log 2 "XML data mismatch, expected '$2', actual '$xml_val'"
+ return 1
+ fi
+ return 0
+}
\ No newline at end of file