diff --git a/tests/.env.docker.default b/tests/.env.docker.default index 0ea1637..2d4187a 100644 --- a/tests/.env.docker.default +++ b/tests/.env.docker.default @@ -25,4 +25,5 @@ USERNAME_TWO=HIJKLMN PASSWORD_TWO=OPQRSTU TEST_FILE_FOLDER=$PWD/versity-gwtest-files RECREATE_BUCKETS=true -REMOVE_TEST_FILE_FOLDER=true \ No newline at end of file +REMOVE_TEST_FILE_FOLDER=true +VERSIONING_DIR=/tmp/versioning \ No newline at end of file diff --git a/tests/README.md b/tests/README.md index 8c784d9..95bdfaa 100644 --- a/tests/README.md +++ b/tests/README.md @@ -75,10 +75,11 @@ To communicate directly with s3, in order to compare the gateway results to dire 1. Copy `.secrets.default` to `.secrets` in the `tests` folder and change the parameters and add the additional s3 fields explained in the **S3 Backend** section above if running with the s3 backend. 2. By default, the dockerfile uses the **arm** architecture (usually modern Mac). If using **amd** (usually earlier Mac or Linux), you can either replace the corresponding `ARG` values directly, or with `arg="="` Also, you can determine which is used by your OS with `uname -a`. 3. Build and run the `Dockerfile_test_bats` file. Change the `SECRETS_FILE` and `CONFIG_FILE` parameters to point to your secrets and config file, respectively, if not using the defaults. Example: `docker build -t -f Dockerfile_test_bats --build-arg="SECRETS_FILE=" --build-arg="CONFIG_FILE=" .`. +4. To run the entire suite, run `docker run -it `. To run an individual suite, pass in the name of the suite as defined in `tests/run.sh` (e.g. REST tests -> `docker run -it rest`). Also, multiple specific suites can be run, if separated by comma. ## Instructions - Running with docker-compose -A file named `docker-compose-bats.yml` is provided in the root folder. Four configurations are provided: +A file named `docker-compose-bats.yml` is provided in the root folder. A few configurations are provided, and you can also create your own provided you have a secrets and config file: * insecure (without certificates), with creation/removal of buckets * secure, posix backend, with static buckets * secure, posix backend, with creation/removal of buckets diff --git a/tests/rest_scripts/list_objects.sh b/tests/rest_scripts/list_objects.sh index d39743e..39ef7bc 100755 --- a/tests/rest_scripts/list_objects.sh +++ b/tests/rest_scripts/list_objects.sh @@ -20,24 +20,55 @@ source ./tests/rest_scripts/rest.sh # shellcheck disable=SC2153 bucket_name="$BUCKET_NAME" +version_two="${VERSION_TWO:-FALSE}" +max_keys="${MAX_KEYS:-0}" +# shellcheck disable=SC2153 +if [ "$CONTINUATION_TOKEN" != "" ]; then + continuation_token=$(jq -rn --arg token "$CONTINUATION_TOKEN" '$token | @uri') +fi current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") #x-amz-object-attributes:ETag canonical_request="GET /$bucket_name +" +if [ "$CONTINUATION_TOKEN" != "" ]; then + add_parameter "canonical_request" "continuation-token=$continuation_token" +fi +if [ "$version_two" != "FALSE" ]; then + add_parameter "canonical_request" "list-type=2" +fi +if [ "$max_keys" -ne 0 ]; then + add_parameter "canonical_request" "max-keys=$max_keys" +fi +first_param_added="false" + +canonical_request+=" host:$host x-amz-content-sha256:UNSIGNED-PAYLOAD x-amz-date:$current_date_time host;x-amz-content-sha256;x-amz-date UNSIGNED-PAYLOAD" - create_canonical_hash_sts_and_signature -curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name" --H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +curl_command+=(curl -ks -w "\"%{http_code}\"") +url="'$AWS_ENDPOINT_URL/$bucket_name" +if [ "$CONTINUATION_TOKEN" != "" ]; then + add_parameter "url" "continuation-token=$continuation_token" +fi +if [ "$version_two" != "FALSE" ]; then + add_parameter "url" "list-type=2" +fi +if [ "$max_keys" -ne 0 ]; then + add_parameter "url" "max-keys=$max_keys" +fi +first_param_added="false" +url+="'" +curl_command+=("$url") +curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" -H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" -H "\"x-amz-date: $current_date_time\"" -o "$OUTPUT_FILE") diff --git a/tests/rest_scripts/rest.sh b/tests/rest_scripts/rest.sh index eb20ea0..332de21 100644 --- a/tests/rest_scripts/rest.sh +++ b/tests/rest_scripts/rest.sh @@ -65,3 +65,18 @@ $canonical_request_hash" curl_command=() add_command_recording_if_enabled } + +add_parameter() { + if [ "$#" -ne 2 ]; then + return + fi + if [ "$first_param_added" != "true" ]; then + if [ "$1" == "url" ]; then + eval "$1"+='?' + fi + eval "$1"+="$2" + first_param_added="true" + else + eval "$1"+="'&$2'" + fi +} diff --git a/tests/test_rest.sh b/tests/test_rest.sh index a7a2b64..5cba21b 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -43,6 +43,7 @@ source ./tests/util/util_rest.sh source ./tests/util/util_tags.sh source ./tests/util/util_time.sh source ./tests/util/util_versioning.sh +source ./tests/util/util_xml.sh export RUN_USERS=true @@ -437,3 +438,34 @@ export RUN_USERS=true run put_and_check_policy_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/policy_file.txt" "Allow" "$USERNAME_ONE" "s3:PutBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME" assert_success } + +@test "REST - list objects v2 - invalid continuation token" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/993" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test_file" + test_file_two="test_file_2" + test_file_three="test_file_3" + run create_test_files "$test_file" "$test_file_two" "$test_file_three" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_two" "$BUCKET_ONE_NAME" "$test_file_two" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_three" "$BUCKET_ONE_NAME" "$test_file_three" + assert_success + + run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" + assert_success + continuation_token=$output + + # interestingly, AWS appears to accept continuation tokens that are a few characters off, so have to remove three chars + run list_objects_check_continuation_error "$BUCKET_ONE_NAME" "${continuation_token:0:${#continuation_token}-3}" + assert_success +} diff --git a/tests/util/util_list_objects.sh b/tests/util/util_list_objects.sh index 08cb7b0..837311a 100644 --- a/tests/util/util_list_objects.sh +++ b/tests/util/util_list_objects.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash source ./tests/commands/list_objects_v2.sh +source ./tests/util/util_xml.sh # Copyright 2024 Versity Software # This file is licensed under the Apache License, Version 2.0 @@ -242,4 +243,61 @@ list_objects_with_user_rest_verify_success() { return 1 fi return 0 +} + +list_objects_check_params_get_token() { + if [ $# -ne 3 ]; then + log 2 "'list_objects_check_params_get_token' requires bucket name, files" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then + log 2 "error attempting to get bucket ACL response: $result" + return 1 + fi + log 5 "objects: $(cat "$TEST_FILE_FOLDER/objects.txt")" + if ! list_bucket_result=$(xmllint --xpath '//*[local-name()="ListBucketResult"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then + log 2 "error getting list bucket result: $list_bucket_result" + return 1 + fi + if ! check_xml_element <(echo "$list_bucket_result") "$2" "Key"; then + log 2 "key mismatch" + return 1 + fi + if ! check_xml_element <(echo "$list_bucket_result") "1" "MaxKeys"; then + log 2 "max keys mismatch" + return 1 + fi + if ! check_xml_element <(echo "$list_bucket_result") "1" "KeyCount"; then + log 2 "key count mismatch" + return 1 + fi + if ! check_xml_element <(echo "$list_bucket_result") "true" "IsTruncated"; then + log 2 "key count mismatch" + return 1 + fi + if ! continuation_token=$(xmllint --xpath '//*[local-name()="NextContinuationToken"]/text()' <(echo "$list_bucket_result") 2>&1); then + log 2 "error getting next continuation token: $continuation_token" + return 1 + fi + echo "$continuation_token" + return 0 +} + +list_objects_check_continuation_error() { + if [ $# -ne 2 ]; then + log 2 "'list_objects_check_continuation_error' requires bucket name, continuation token" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 CONTINUATION_TOKEN="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then + log 2 "error attempting to get bucket ACL response: $result" + return 1 + fi + if [ "$result" != "400" ]; then + log 2 "expected result code of '400' was '$result'" + return 1 + fi + if ! check_xml_element "$TEST_FILE_FOLDER/objects.txt" "InvalidArgument" "Error" "Code"; then + log 2 "invalid error code" + return 1 + fi } \ No newline at end of file diff --git a/tests/util/util_xml.sh b/tests/util/util_xml.sh new file mode 100644 index 0000000..1cad8c2 --- /dev/null +++ b/tests/util/util_xml.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +check_xml_element() { + if [ $# -lt 3 ]; then + log 2 "'check_xml_element' requires data source, expected value, XML tree" + return 1 + fi + local xpath='//' + for tree_val in "${@:3}"; do + xpath+='*[local-name()="'$tree_val'"]/' + done + xpath+='text()' + if ! xml_val=$(xmllint --xpath "$xpath" "$1" 2>&1); then + log 2 "error getting XML value matching $xpath: $xml_val" + return 1 + fi + if [ "$2" != "$xml_val" ]; then + log 2 "XML data mismatch, expected '$2', actual '$xml_val'" + return 1 + fi + return 0 +} \ No newline at end of file