From 7975b9bbaa3c4954883e39c76059d419178a72eb Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Wed, 11 Mar 2026 11:07:06 -0300 Subject: [PATCH] test: listobjects delimiter/prefix test, skips removal, go query improvement --- tests/drivers/file.sh | 35 ++++++++ .../drivers/list_objects/list_objects_rest.sh | 83 +++++++++++++++++++ tests/drivers/xml.sh | 53 ++++++++++-- tests/env.sh | 4 +- tests/rest_scripts/command/canonicalQuery.go | 76 +++++++++++++++++ tests/rest_scripts/command/s3Command.go | 26 ++++-- tests/templates/matrix.yaml | 4 +- tests/test_rest_cors.sh | 3 - tests/test_rest_head_object.sh | 3 - tests/test_rest_list_buckets.sh | 5 +- tests/test_rest_list_object_versions.sh | 3 - tests/test_rest_list_objects.sh | 22 +++++ tests/test_rest_not_implemented.sh | 6 -- tests/test_rest_put_object.sh | 10 +-- 14 files changed, 294 insertions(+), 39 deletions(-) create mode 100644 tests/rest_scripts/command/canonicalQuery.go diff --git a/tests/drivers/file.sh b/tests/drivers/file.sh index ddd65236..54087618 100644 --- a/tests/drivers/file.sh +++ b/tests/drivers/file.sh @@ -169,3 +169,38 @@ get_file_name_with_prefix() { fi echo "$1-${uuid}" } + +create_test_files_and_folders() { + if ! check_param_count_gt "any test files, including directories" 1 $#; then + return 1 + fi + + local file="" + local dir="" + local err="" + for file in "$@"; do + if ! create_folder_if_needed_and_file "$file"; then + log 2 "error creating folder if needed and file" + return 1 + fi + done + return 0 +} + +create_folder_if_needed_and_file() { + if ! check_param_count_v2 "file path" 1 $#; then + return 1 + fi + dir=$(dirname "$1") + if [ "$dir" != "." ] && [ ! -d "$TEST_FILE_FOLDER/$dir" ]; then + if ! err=$(mkdir -p "$TEST_FILE_FOLDER/$dir" 2>&1); then + log 2 "error creating folder '$dir': $err" + return 1 + fi + fi + if ! create_test_file "$file"; then + log 2 "error creating test file '$file'" + return 1 + fi + return 0 +} diff --git a/tests/drivers/list_objects/list_objects_rest.sh b/tests/drivers/list_objects/list_objects_rest.sh index b3e7553a..d94ef934 100644 --- a/tests/drivers/list_objects/list_objects_rest.sh +++ b/tests/drivers/list_objects/list_objects_rest.sh @@ -80,3 +80,86 @@ list_check_objects_rest_v2() { return 1 fi } + +check_single_common_prefix_or_key() { + if ! check_param_count_v2 "data file, parameter, prefix or not" 3 $#; then + return 1 + fi + if [ "$3" == "true" ]; then + if ! check_if_element_exists "$1" "$2" "ListBucketResult" "CommonPrefixes" "Prefix"; then + log 2 "error checking if CommonPrefix '$2' exists" + return 1 + fi + else + if ! check_if_element_exists "$1" "$2" "ListBucketResult" "Contents" "Key"; then + log 2 "error checking if Key '$2' exists" + return 1 + fi + fi + return 0 +} + +check_common_prefixes_and_keys() { + if ! check_param_count_gt "data file, prefix, delimiter, common prefixes, --, keys" 4 $#; then + return 1 + fi + if ! xml_data=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then + log 2 "error parsing xml data: $xml_data" + return 1 + fi + local checking_prefixes="true" prefix_count=0 key_count=0 + for param in "${@:4}"; do + if [ "$param" == "--" ]; then + checking_prefixes=false + continue + fi + if ! check_single_common_prefix_or_key "$xml_data" "$param" "$checking_prefixes"; then + log 2 "error checking if common prefix or key '$param' exists" + return 1 + fi + if [ "$checking_prefixes" == "true" ]; then + ((prefix_count++)) + else + ((key_count++)) + fi + done + if ! check_prefix_delimiter_and_counts "$xml_data" "$2" "$3" "$prefix_count" "$key_count"; then + log 2 "error checking prefix" + return 1 + fi + return 0 +} + +check_prefix_delimiter_and_counts() { + if ! check_param_count_v2 "data, prefix, delimiter, prefix count, key count" 5 $#; then + return 1 + fi + if ! check_xml_element "$1" "$2" "ListBucketResult" "Prefix"; then + log 2 "error checking prefix" + return 1 + fi + if ! check_xml_element "$1" "$3" "ListBucketResult" "Delimiter"; then + log 2 "error checking delimiter" + return 1 + fi + if ! check_element_count "$1" "$4" "ListBucketResult" "CommonPrefixes" "Prefix"; then + log 2 "Prefix count mismatch" + return 1 + fi + if ! check_element_count "$1" "$5" "ListBucketResult" "Contents" "Key"; then + log 2 "Prefix count mismatch" + return 1 + fi + return 0 +} + +list_objects_with_prefix_and_delimiter_check_results() { + if ! check_param_count_gt "bucket name, prefix, delimiter, expected common prefixes, --, expected keys" 5 $#; then + return 1 + fi + if ! send_rest_go_command_callback "200" "check_common_prefixes_and_keys" "-bucketName" "$1" "-query" "delimiter=$3&prefix=$2" "--" "${@:2}"; then + log 2 "error sending command to list objects or receiving response" + return 1 + fi + return 0 +} diff --git a/tests/drivers/xml.sh b/tests/drivers/xml.sh index d8eee1f2..8794965e 100644 --- a/tests/drivers/xml.sh +++ b/tests/drivers/xml.sh @@ -242,19 +242,18 @@ check_if_element_exists() { return 1 fi - log 5 "data: $(cat "$1")" - log 5 "xpath: $xpath" - if ! get_xml_data "$1" "$1.xml"; then - log 2 "error getting XML data" + if ! data_file=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then + log 2 "error checking XML data: $data_file" return 1 fi - if ! result=$(xmllint --xpath "boolean(${xpath}[text()='$2'])" "$1.xml" 2>&1); then + if ! result=$(xmllint --xpath "boolean(${xpath}[text()='$2'])" "$data_file" 2>&1); then log 2 "error getting result: $result" return 1 fi if [ "$result" == "true" ]; then return 0 fi + log 5 "element '$2' not found" return 1 } @@ -393,3 +392,47 @@ get_element_with_matching_inner_value() { echo "$result" return 0 } + +check_validity_and_or_parse_xml_data() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + if xmllint --noout "$1" 2>/dev/null; then + echo "$1" + return 0 + fi + if ! filtered_xml_file=$(get_file_name 2>&1); then + log 2 "error getting file name: $filtered_xml_file" + return 1 + fi + if ! get_xml_data "$1" "$TEST_FILE_FOLDER/$filtered_xml_file"; then + log 2 "error getting XML data" + return 1 + fi + log 5 "filtered data: $(cat "$TEST_FILE_FOLDER/$filtered_xml_file")" + echo "$TEST_FILE_FOLDER/$filtered_xml_file" + return 0 +} + +check_element_count() { + if ! check_param_count_gt "data file, expected count, XML tree" 2 $#; then + return 1 + fi + if ! xpath=$(build_xpath_string_for_element "${@:3}" 2>&1); then + log 2 "error building xpath string: $xpath" + return 1 + fi + if ! data_file=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then + log 2 "error getting XML data: $data_file" + return 1 + fi + if ! count=$(xmllint --xpath "count($xpath)" "$data_file" 2>&1); then + log 2 "error getting element '$xpath' count: $count" + return 1 + fi + if [ "$count" != "$2" ]; then + log 2 "expected count of '$2', was '$count'" + return 1 + fi + return 0 +} diff --git a/tests/env.sh b/tests/env.sh index 58571abf..bf4cd39d 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -107,10 +107,10 @@ check_aws_vars() { log 1 "AWS_ENDPOINT_URL missing" exit 1 fi - export SERVER_NAME="VERSITYGW" + export SERVER_NAME="versitygw" else if [ -z "$SERVER_NAME" ]; then - export SERVER_NAME="AMAZONS3" + export SERVER_NAME="amazonS3" else export SERVER_NAME fi diff --git a/tests/rest_scripts/command/canonicalQuery.go b/tests/rest_scripts/command/canonicalQuery.go new file mode 100644 index 00000000..9bd843c1 --- /dev/null +++ b/tests/rest_scripts/command/canonicalQuery.go @@ -0,0 +1,76 @@ +package command + +import ( + "fmt" + "net/url" + "sort" + "strings" +) + +type queryPair struct { + key string + value string +} + +// awsQueryEscape applies the AWS SigV4 percent-encoding rules. +// - Spaces must be encoded as %20 (not '+') +// - '~' must not be escaped +func awsQueryEscape(s string) string { + esc := url.QueryEscape(s) + esc = strings.ReplaceAll(esc, "+", "%20") + esc = strings.ReplaceAll(esc, "%7E", "~") + return esc +} + +// canonicalizeQuery converts a raw query string into an AWS SigV4 canonical query string. +// It percent-encodes keys/values, sorts them, and joins as k=v pairs. +func canonicalizeQuery(raw string) (string, error) { + if raw == "" { + return "", nil + } + + // Treat bare subresource values like "cors" as "cors=". + if !strings.Contains(raw, "=") && !strings.HasSuffix(raw, "=") { + raw += "=" + } + + vals, err := url.ParseQuery(raw) + if err != nil { + return "", fmt.Errorf("error parsing query: %w", err) + } + + pairs := getQueryPairs(vals) + sort.Slice(pairs, func(i, j int) bool { + escapedKeyI, escapedKeyJ := awsQueryEscape(pairs[i].key), awsQueryEscape(pairs[j].key) + if escapedKeyI != escapedKeyJ { + return escapedKeyI < escapedKeyJ + } + escapedValueI, escapedValueJ := awsQueryEscape(pairs[i].value), awsQueryEscape(pairs[j].value) + return escapedValueI < escapedValueJ + }) + + var b strings.Builder + for i, p := range pairs { + if i > 0 { + b.WriteByte('&') + } + b.WriteString(awsQueryEscape(p.key)) + b.WriteByte('=') + b.WriteString(awsQueryEscape(p.value)) + } + return b.String(), nil +} + +func getQueryPairs(values url.Values) []queryPair { + pairs := make([]queryPair, 0, len(values)) + for queryKey, queryValues := range values { + if len(queryValues) == 0 { + pairs = append(pairs, queryPair{key: queryKey, value: ""}) + continue + } + for _, v := range queryValues { + pairs = append(pairs, queryPair{key: queryKey, value: v}) + } + } + return pairs +} diff --git a/tests/rest_scripts/command/s3Command.go b/tests/rest_scripts/command/s3Command.go index 9f326c2d..00b21821 100644 --- a/tests/rest_scripts/command/s3Command.go +++ b/tests/rest_scripts/command/s3Command.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "net/url" "os" "sort" "strings" @@ -309,7 +310,12 @@ func (s *S3Command) generateCanonicalRequestString() { if queryRequestLine == "" { queryRequestLine = s.Query } - canonicalRequestLines = append(canonicalRequestLines, queryRequestLine) + canonicalQuery, err := canonicalizeQuery(queryRequestLine) + if err != nil { + logger.PrintDebug("error parsing query '%s': %v", queryRequestLine, err) + canonicalQuery = queryRequestLine + } + canonicalRequestLines = append(canonicalRequestLines, canonicalQuery) var signedParams []string for _, headerValue := range s.headerValues { @@ -373,12 +379,20 @@ func (s *S3Command) buildCurlShellCommand() (string, error) { if s.Method != "GET" { curlCommand = append(curlCommand, fmt.Sprintf("-X %s ", s.Method)) } - fullPath := "\"" + s.Url + s.path - if s.Query != "" { - fullPath += "?" + s.Query + fullPath := s.Url + s.path + awsUrl, err := url.Parse(fullPath) + if err != nil { + return "", fmt.Errorf("error parsing URL: %w", err) } - fullPath += "\"" - curlCommand = append(curlCommand, fullPath) + if s.Query != "" { + canonicalQuery, err := canonicalizeQuery(s.Query) + if err != nil { + return "", fmt.Errorf("error parsing query: %w", err) + } + awsUrl.RawQuery = canonicalQuery + } + enclosedPath := fmt.Sprintf("\"%s\"", awsUrl.String()) + curlCommand = append(curlCommand, enclosedPath) authorizationString := s.buildAuthorizationString() curlCommand = append(curlCommand, "-H", fmt.Sprintf("\"%s\"", authorizationString)) for _, headerValue := range s.headerValues { diff --git a/tests/templates/matrix.yaml b/tests/templates/matrix.yaml index 0163955b..6e0f7e49 100644 --- a/tests/templates/matrix.yaml +++ b/tests/templates/matrix.yaml @@ -1,7 +1,7 @@ test_rest_not_implemented.sh: test_REST_-2d_GetBucketAnalyticsConfiguration_-2d_with_template: default: - VERSITYGW: + versitygw: not-implemented.yaml.tmpl - AMAZONS3: + amazonS3: get-bucket-analytics.yaml.tmpl diff --git a/tests/test_rest_cors.sh b/tests/test_rest_cors.sh index 2c65349a..11eb9fb9 100755 --- a/tests/test_rest_cors.sh +++ b/tests/test_rest_cors.sh @@ -101,9 +101,6 @@ source ./tests/setup.sh } @test "REST - CORS - invalid origin" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1870" - fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success bucket_name=$output diff --git a/tests/test_rest_head_object.sh b/tests/test_rest_head_object.sh index 2cbd6d47..a1f02bdb 100755 --- a/tests/test_rest_head_object.sh +++ b/tests/test_rest_head_object.sh @@ -45,9 +45,6 @@ source ./tests/drivers/get_object_attributes/get_object_attributes_rest.sh } @test "REST - HeadObject - default Content-Type is binary/octet-stream" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1849" - fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success bucket_name="$output" diff --git a/tests/test_rest_list_buckets.sh b/tests/test_rest_list_buckets.sh index 5e9349b5..ddd49761 100755 --- a/tests/test_rest_list_buckets.sh +++ b/tests/test_rest_list_buckets.sh @@ -232,10 +232,7 @@ export RUN_USERS=true } @test "REST - ListBuckets - error Content-Type is application/xml" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1852" - fi - run send_rest_go_command_check_header_key_and_value "400" "Content-Type" "application/xml" "-method" "GETS" + run send_rest_go_command_check_header_key_and_value "400" "Content-Type" "application/xml" "-method" "GET" "-omitDate" assert_success } diff --git a/tests/test_rest_list_object_versions.sh b/tests/test_rest_list_object_versions.sh index 0d3de68d..4927daed 100755 --- a/tests/test_rest_list_object_versions.sh +++ b/tests/test_rest_list_object_versions.sh @@ -38,9 +38,6 @@ source ./tests/util/util_time.sh } @test "ListObjectVersions - version changes after deletion w/retention policy" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1741" - fi test_file="test_file" run get_bucket_name "$BUCKET_ONE_NAME" diff --git a/tests/test_rest_list_objects.sh b/tests/test_rest_list_objects.sh index c254fef1..8ba71681 100755 --- a/tests/test_rest_list_objects.sh +++ b/tests/test_rest_list_objects.sh @@ -152,3 +152,25 @@ source ./tests/drivers/create_bucket/create_bucket_rest.sh run list_objects_v1_check_nextmarker_empty "$bucket_name" assert_success } + +@test "REST - ListObjects - delimiter" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + local bucket_name="$output" + + file_names=("a-b-1.txt" "a-b-2.txt" "a-b/c-1.txt" "a-b/c-2.txt" "a-b/d.txt" "a/c.txt") + local prefix="a-" + run create_test_files_and_folders "${file_names[@]}" + assert_success + + run setup_bucket_v2 "$bucket_name" + assert_success + + for file_name in "${file_names[@]}"; do + run put_object "rest" "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + done + + run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt" + assert_success +} diff --git a/tests/test_rest_not_implemented.sh b/tests/test_rest_not_implemented.sh index 0c5e18ed..525b0eea 100755 --- a/tests/test_rest_not_implemented.sh +++ b/tests/test_rest_not_implemented.sh @@ -27,9 +27,6 @@ source ./tests/setup.sh } @test "REST - GetBucketAnalyticsConfiguration - with template" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1821" - fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success bucket_name=$output @@ -50,9 +47,6 @@ source ./tests/setup.sh } @test "REST - NotImplemented - correct Content-Type header" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1821" - fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success bucket_name=$output diff --git a/tests/test_rest_put_object.sh b/tests/test_rest_put_object.sh index 8e21a30b..03d7db23 100755 --- a/tests/test_rest_put_object.sh +++ b/tests/test_rest_put_object.sh @@ -394,7 +394,7 @@ export RUN_USERS=true @test "PutObject - x-amz-acl - not implemented" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1767" + skip "skip for versitygw - see #1904" fi run get_bucket_name "$BUCKET_ONE_NAME" assert_success @@ -422,7 +422,7 @@ export RUN_USERS=true @test "PutObject - x-amz-grant-full-control - not implemented" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1773" + skip "skip for versitygw - see #1904" fi run attempt_put_object_with_specific_acl "x-amz-grant-full-control" assert_success @@ -430,14 +430,14 @@ export RUN_USERS=true @test "PutObject - x-amz-grant-read - not implemented" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1773" + skip "skip for versitygw - see #1904" fi run attempt_put_object_with_specific_acl "x-amz-grant-read" assert_success } @test "PutObject - x-amz-grant-read-acp - not implemented" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1773" + skip "skip for versitygw - see #1904" fi run attempt_put_object_with_specific_acl "x-amz-grant-read-acp" assert_success @@ -445,7 +445,7 @@ export RUN_USERS=true @test "PutObject - x-amz-grant-write-acp - not implemented" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1773" + skip "skip for versitygw - see #1904" fi run attempt_put_object_with_specific_acl "x-amz-grant-write-acp" assert_success