test: listobjects delimiter/prefix test, skips removal, go query improvement

tests/drivers/file.sh

@@ -169,3 +169,38 @@ get_file_name_with_prefix() {
   fi
   echo "$1-${uuid}"
 }
+
+create_test_files_and_folders() {
+  if ! check_param_count_gt "any test files, including directories" 1 $#; then
+    return 1
+  fi
+
+  local file=""
+  local dir=""
+  local err=""
+  for file in "$@"; do
+    if ! create_folder_if_needed_and_file "$file"; then
+      log 2 "error creating folder if needed and file"
+      return 1
+    fi
+  done
+  return 0
+}
+
+create_folder_if_needed_and_file() {
+  if ! check_param_count_v2 "file path" 1 $#; then
+    return 1
+  fi
+  dir=$(dirname "$1")
+  if [ "$dir" != "." ] && [ ! -d "$TEST_FILE_FOLDER/$dir" ]; then
+    if ! err=$(mkdir -p "$TEST_FILE_FOLDER/$dir" 2>&1); then
+      log 2 "error creating folder '$dir': $err"
+      return 1
+    fi
+  fi
+  if ! create_test_file "$file"; then
+    log 2 "error creating test file '$file'"
+    return 1
+  fi
+  return 0
+}

tests/drivers/list_objects/list_objects_rest.sh

@@ -80,3 +80,86 @@ list_check_objects_rest_v2() {
     return 1
   fi
 }
+
+check_single_common_prefix_or_key() {
+  if ! check_param_count_v2 "data file, parameter, prefix or not" 3 $#; then
+    return 1
+  fi
+  if [ "$3" == "true" ]; then
+    if ! check_if_element_exists "$1" "$2" "ListBucketResult" "CommonPrefixes" "Prefix"; then
+      log 2 "error checking if CommonPrefix '$2' exists"
+      return 1
+    fi
+  else
+    if ! check_if_element_exists "$1" "$2" "ListBucketResult" "Contents" "Key"; then
+      log 2 "error checking if Key '$2' exists"
+      return 1
+    fi
+  fi
+  return 0
+}
+
+check_common_prefixes_and_keys() {
+  if ! check_param_count_gt "data file, prefix, delimiter, common prefixes, --, keys" 4 $#; then
+    return 1
+  fi
+  if ! xml_data=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then
+    log 2 "error parsing xml data: $xml_data"
+    return 1
+  fi
+  local checking_prefixes="true" prefix_count=0 key_count=0
+  for param in "${@:4}"; do
+    if [ "$param" == "--" ]; then
+      checking_prefixes=false
+      continue
+    fi
+    if ! check_single_common_prefix_or_key "$xml_data" "$param" "$checking_prefixes"; then
+      log 2 "error checking if common prefix or key '$param' exists"
+      return 1
+    fi
+    if [ "$checking_prefixes" == "true" ]; then
+      ((prefix_count++))
+    else
+      ((key_count++))
+    fi
+  done
+  if ! check_prefix_delimiter_and_counts "$xml_data" "$2" "$3" "$prefix_count" "$key_count"; then
+    log 2 "error checking prefix"
+    return 1
+  fi
+  return 0
+}
+
+check_prefix_delimiter_and_counts() {
+  if ! check_param_count_v2 "data, prefix, delimiter, prefix count, key count" 5 $#; then
+    return 1
+  fi
+  if ! check_xml_element "$1" "$2" "ListBucketResult" "Prefix"; then
+    log 2 "error checking prefix"
+    return 1
+  fi
+  if ! check_xml_element "$1" "$3" "ListBucketResult" "Delimiter"; then
+    log 2 "error checking delimiter"
+    return 1
+  fi
+  if ! check_element_count "$1" "$4" "ListBucketResult" "CommonPrefixes" "Prefix"; then
+    log 2 "Prefix count mismatch"
+    return 1
+  fi
+  if ! check_element_count "$1" "$5" "ListBucketResult" "Contents" "Key"; then
+    log 2 "Prefix count mismatch"
+    return 1
+  fi
+  return 0
+}
+
+list_objects_with_prefix_and_delimiter_check_results() {
+  if ! check_param_count_gt "bucket name, prefix, delimiter, expected common prefixes, --, expected keys" 5 $#; then
+    return 1
+  fi
+  if ! send_rest_go_command_callback "200" "check_common_prefixes_and_keys" "-bucketName" "$1" "-query" "delimiter=$3&prefix=$2" "--" "${@:2}"; then
+    log 2 "error sending command to list objects or receiving response"
+    return 1
+  fi
+  return 0
+}

tests/drivers/xml.sh

@@ -242,19 +242,18 @@ check_if_element_exists() {
     return 1
   fi
 
-  log 5 "data: $(cat "$1")"
-  log 5 "xpath: $xpath"
-  if ! get_xml_data "$1" "$1.xml"; then
-    log 2 "error getting XML data"
+  if ! data_file=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then
+    log 2 "error checking XML data: $data_file"
     return 1
   fi
-  if ! result=$(xmllint --xpath "boolean(${xpath}[text()='$2'])" "$1.xml" 2>&1); then
+  if ! result=$(xmllint --xpath "boolean(${xpath}[text()='$2'])" "$data_file" 2>&1); then
     log 2 "error getting result: $result"
     return 1
   fi
   if [ "$result" == "true" ]; then
     return 0
   fi
+  log 5 "element '$2' not found"
   return 1
 }
 
@@ -393,3 +392,47 @@ get_element_with_matching_inner_value() {
   echo "$result"
   return 0
 }
+
+check_validity_and_or_parse_xml_data() {
+  if ! check_param_count_v2 "data file" 1 $#; then
+    return 1
+  fi
+  if xmllint --noout "$1" 2>/dev/null; then
+    echo "$1"
+    return 0
+  fi
+  if ! filtered_xml_file=$(get_file_name 2>&1); then
+    log 2 "error getting file name: $filtered_xml_file"
+    return 1
+  fi
+  if ! get_xml_data "$1" "$TEST_FILE_FOLDER/$filtered_xml_file"; then
+    log 2 "error getting XML data"
+    return 1
+  fi
+  log 5 "filtered data: $(cat "$TEST_FILE_FOLDER/$filtered_xml_file")"
+  echo "$TEST_FILE_FOLDER/$filtered_xml_file"
+  return 0
+}
+
+check_element_count() {
+  if ! check_param_count_gt "data file, expected count, XML tree" 2 $#; then
+    return 1
+  fi
+  if ! xpath=$(build_xpath_string_for_element "${@:3}" 2>&1); then
+    log 2 "error building xpath string: $xpath"
+    return 1
+  fi
+  if ! data_file=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then
+    log 2 "error getting XML data: $data_file"
+    return 1
+  fi
+  if ! count=$(xmllint --xpath "count($xpath)" "$data_file" 2>&1); then
+    log 2 "error getting element '$xpath' count: $count"
+    return 1
+  fi
+  if [ "$count" != "$2" ]; then
+    log 2 "expected count of '$2', was '$count'"
+    return 1
+  fi
+  return 0
+}

tests/env.sh

@@ -107,10 +107,10 @@ check_aws_vars() {
       log 1 "AWS_ENDPOINT_URL missing"
       exit 1
     fi
-    export SERVER_NAME="VERSITYGW"
+    export SERVER_NAME="versitygw"
   else
     if [ -z "$SERVER_NAME" ]; then
-      export SERVER_NAME="AMAZONS3"
+      export SERVER_NAME="amazonS3"
     else
       export SERVER_NAME
     fi

tests/rest_scripts/command/canonicalQuery.go

@@ -0,0 +1,76 @@
+package command
+
+import (
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+)
+
+type queryPair struct {
+	key   string
+	value string
+}
+
+// awsQueryEscape applies the AWS SigV4 percent-encoding rules.
+// - Spaces must be encoded as %20 (not '+')
+// - '~' must not be escaped
+func awsQueryEscape(s string) string {
+	esc := url.QueryEscape(s)
+	esc = strings.ReplaceAll(esc, "+", "%20")
+	esc = strings.ReplaceAll(esc, "%7E", "~")
+	return esc
+}
+
+// canonicalizeQuery converts a raw query string into an AWS SigV4 canonical query string.
+// It percent-encodes keys/values, sorts them, and joins as k=v pairs.
+func canonicalizeQuery(raw string) (string, error) {
+	if raw == "" {
+		return "", nil
+	}
+
+	// Treat bare subresource values like "cors" as "cors=".
+	if !strings.Contains(raw, "=") && !strings.HasSuffix(raw, "=") {
+		raw += "="
+	}
+
+	vals, err := url.ParseQuery(raw)
+	if err != nil {
+		return "", fmt.Errorf("error parsing query: %w", err)
+	}
+
+	pairs := getQueryPairs(vals)
+	sort.Slice(pairs, func(i, j int) bool {
+		escapedKeyI, escapedKeyJ := awsQueryEscape(pairs[i].key), awsQueryEscape(pairs[j].key)
+		if escapedKeyI != escapedKeyJ {
+			return escapedKeyI < escapedKeyJ
+		}
+		escapedValueI, escapedValueJ := awsQueryEscape(pairs[i].value), awsQueryEscape(pairs[j].value)
+		return escapedValueI < escapedValueJ
+	})
+
+	var b strings.Builder
+	for i, p := range pairs {
+		if i > 0 {
+			b.WriteByte('&')
+		}
+		b.WriteString(awsQueryEscape(p.key))
+		b.WriteByte('=')
+		b.WriteString(awsQueryEscape(p.value))
+	}
+	return b.String(), nil
+}
+
+func getQueryPairs(values url.Values) []queryPair {
+	pairs := make([]queryPair, 0, len(values))
+	for queryKey, queryValues := range values {
+		if len(queryValues) == 0 {
+			pairs = append(pairs, queryPair{key: queryKey, value: ""})
+			continue
+		}
+		for _, v := range queryValues {
+			pairs = append(pairs, queryPair{key: queryKey, value: v})
+		}
+	}
+	return pairs
+}

tests/rest_scripts/command/s3Command.go

@@ -7,6 +7,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
+	"net/url"
 	"os"
 	"sort"
 	"strings"
@@ -309,7 +310,12 @@ func (s *S3Command) generateCanonicalRequestString() {
 	if queryRequestLine == "" {
 		queryRequestLine = s.Query
 	}
-	canonicalRequestLines = append(canonicalRequestLines, queryRequestLine)
+	canonicalQuery, err := canonicalizeQuery(queryRequestLine)
+	if err != nil {
+		logger.PrintDebug("error parsing query '%s': %v", queryRequestLine, err)
+		canonicalQuery = queryRequestLine
+	}
+	canonicalRequestLines = append(canonicalRequestLines, canonicalQuery)
 
 	var signedParams []string
 	for _, headerValue := range s.headerValues {
@@ -373,12 +379,20 @@ func (s *S3Command) buildCurlShellCommand() (string, error) {
 	if s.Method != "GET" {
 		curlCommand = append(curlCommand, fmt.Sprintf("-X %s ", s.Method))
 	}
-	fullPath := "\"" + s.Url + s.path
-	if s.Query != "" {
-		fullPath += "?" + s.Query
+	fullPath := s.Url + s.path
+	awsUrl, err := url.Parse(fullPath)
+	if err != nil {
+		return "", fmt.Errorf("error parsing URL: %w", err)
 	}
-	fullPath += "\""
-	curlCommand = append(curlCommand, fullPath)
+	if s.Query != "" {
+		canonicalQuery, err := canonicalizeQuery(s.Query)
+		if err != nil {
+			return "", fmt.Errorf("error parsing query: %w", err)
+		}
+		awsUrl.RawQuery = canonicalQuery
+	}
+	enclosedPath := fmt.Sprintf("\"%s\"", awsUrl.String())
+	curlCommand = append(curlCommand, enclosedPath)
 	authorizationString := s.buildAuthorizationString()
 	curlCommand = append(curlCommand, "-H", fmt.Sprintf("\"%s\"", authorizationString))
 	for _, headerValue := range s.headerValues {

tests/templates/matrix.yaml

@@ -1,7 +1,7 @@
 test_rest_not_implemented.sh:
   test_REST_-2d_GetBucketAnalyticsConfiguration_-2d_with_template:
     default:
-      VERSITYGW:
+      versitygw:
         not-implemented.yaml.tmpl
-      AMAZONS3:
+      amazonS3:
         get-bucket-analytics.yaml.tmpl

tests/test_rest_cors.sh

@@ -101,9 +101,6 @@ source ./tests/setup.sh
 }
 
 @test "REST - CORS - invalid origin" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1870"
-  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name=$output

tests/test_rest_head_object.sh

@@ -45,9 +45,6 @@ source ./tests/drivers/get_object_attributes/get_object_attributes_rest.sh
 }
 
 @test "REST - HeadObject - default Content-Type is binary/octet-stream" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1849"
-  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name="$output"

tests/test_rest_list_buckets.sh

@@ -232,10 +232,7 @@ export RUN_USERS=true
 }
 
 @test "REST - ListBuckets - error Content-Type is application/xml" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1852"
-  fi
-  run send_rest_go_command_check_header_key_and_value "400" "Content-Type" "application/xml" "-method" "GETS"
+  run send_rest_go_command_check_header_key_and_value "400" "Content-Type" "application/xml" "-method" "GET" "-omitDate"
   assert_success
 }
 

tests/test_rest_list_object_versions.sh

@@ -38,9 +38,6 @@ source ./tests/util/util_time.sh
 }
 
 @test "ListObjectVersions - version changes after deletion w/retention policy" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1741"
-  fi
   test_file="test_file"
 
   run get_bucket_name "$BUCKET_ONE_NAME"

tests/test_rest_list_objects.sh

@@ -152,3 +152,25 @@ source ./tests/drivers/create_bucket/create_bucket_rest.sh
   run list_objects_v1_check_nextmarker_empty "$bucket_name"
   assert_success
 }
+
+@test "REST - ListObjects - delimiter" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  local bucket_name="$output"
+
+  file_names=("a-b-1.txt" "a-b-2.txt" "a-b/c-1.txt" "a-b/c-2.txt" "a-b/d.txt" "a/c.txt")
+  local prefix="a-"
+  run create_test_files_and_folders "${file_names[@]}"
+  assert_success
+
+  run setup_bucket_v2 "$bucket_name"
+  assert_success
+
+  for file_name in "${file_names[@]}"; do
+    run put_object "rest" "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
+    assert_success
+  done
+
+  run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt"
+  assert_success
+}

tests/test_rest_not_implemented.sh

@@ -27,9 +27,6 @@ source ./tests/setup.sh
 }
 
 @test "REST - GetBucketAnalyticsConfiguration - with template" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1821"
-  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name=$output
@@ -50,9 +47,6 @@ source ./tests/setup.sh
 }
 
 @test "REST - NotImplemented - correct Content-Type header" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1821"
-  fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
   bucket_name=$output

tests/test_rest_put_object.sh

@@ -394,7 +394,7 @@ export RUN_USERS=true
 
 @test "PutObject - x-amz-acl - not implemented" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1767"
+    skip "skip for versitygw - see #1904"
   fi
   run get_bucket_name "$BUCKET_ONE_NAME"
   assert_success
@@ -422,7 +422,7 @@ export RUN_USERS=true
 
 @test "PutObject - x-amz-grant-full-control - not implemented" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1773"
+    skip "skip for versitygw - see #1904"
   fi
   run attempt_put_object_with_specific_acl "x-amz-grant-full-control"
   assert_success
@@ -430,14 +430,14 @@ export RUN_USERS=true
 
 @test "PutObject - x-amz-grant-read - not implemented" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1773"
+    skip "skip for versitygw - see #1904"
   fi
   run attempt_put_object_with_specific_acl "x-amz-grant-read"
   assert_success
 }
 @test "PutObject - x-amz-grant-read-acp - not implemented" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1773"
+    skip "skip for versitygw - see #1904"
   fi
   run attempt_put_object_with_specific_acl "x-amz-grant-read-acp"
   assert_success
@@ -445,7 +445,7 @@ export RUN_USERS=true
 
 @test "PutObject - x-amz-grant-write-acp - not implemented" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1773"
+    skip "skip for versitygw - see #1904"
   fi
   run attempt_put_object_with_specific_acl "x-amz-grant-write-acp"
   assert_success