test: HeadObject query tests, more util code changes

tests/commands/list_buckets.sh

@@ -50,6 +50,8 @@ list_buckets() {
     bucket_name=$(echo "$line" | awk '{print $NF}')
     bucket_array+=("${bucket_name%/}")
   done <<< "$buckets"
+
+  echo "${bucket_array[*]}"
   return 0
 }
 
@@ -86,6 +88,8 @@ list_buckets_with_user() {
     bucket_name=$(echo "$line" | awk '{print $NF}')
     bucket_array+=("${bucket_name%/}")
   done <<< "$buckets"
+
+  echo "${bucket_array[*]}"
   return 0
 }
 
@@ -111,6 +115,7 @@ list_buckets_s3api() {
   names=$(jq -r '.Buckets[].Name' <<<"$modified_output")
   IFS=$'\n' read -rd '' -a bucket_array <<<"$names"
 
+  echo "${bucket_array[*]}"
   return 0
 }
 

tests/drivers/head_object/head_object_rest.sh

@@ -236,3 +236,21 @@ head_object_check_header_key_and_value() {
   fi
   return 0
 }
+
+check_header_partial_content_response() {
+  if ! check_param_count_v2 "header data, part number, full object size, part size" 4 $#; then
+    return 1
+  fi
+  log 5 "header: $(cat "$1")"
+  starting_byte=$((($2-1)*$4))
+  ending_byte=$(($2*$4-1))
+  if [ "$3" -lt "$ending_byte" ]; then
+    ending_byte="$(($3-1))"
+  fi
+  content_range_string="bytes $starting_byte-$ending_byte/$3"
+  if ! result=$(check_for_header_key_and_value "$1" "Content-Range" "$content_range_string" 2>&1); then
+    log 2 "error checking for header key and value: $result"
+    return 1
+  fi
+  return 0
+}

tests/drivers/list_buckets/list_buckets.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright 2024 Versity Software
+# Copyright 2026 Versity Software
 # This file is licensed under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
@@ -19,17 +19,15 @@ list_and_check_buckets_with_user() {
     log 2 "'list_and_check_buckets' requires client, two bucket names, id, key"
     return 1
   fi
-  if ! list_buckets_with_user "$1" "$4" "$5"; then
-    log 2 "error listing buckets"
+  if ! response=$(list_buckets_with_user "$1" "$4" "$5" 2>&1); then
+    log 2 "error listing buckets: $response"
     return 1
   fi
+  read -r -a bucket_array <<< "$response"
 
   local bucket_one_found=false
   local bucket_two_found=false
-  if [ -z "$bucket_array" ]; then
-    log 2 "bucket_array parameter not exported"
-    return 1
-  fi
+
   log 5 "bucket array: ${bucket_array[*]}"
   for bucket in "${bucket_array[@]}"; do
     if [ "$bucket" == "$2" ] || [ "$bucket" == "s3://$2" ]; then
@@ -59,28 +57,3 @@ list_and_check_buckets() {
   fi
   return 0
 }
-
-list_and_check_buckets_omit_without_permission() {
-  if [ $# -ne 4 ]; then
-    log 2 "'list_and_check_buckets_with_user' requires username, password, non-visible bucket, visible bucket"
-    return 1
-  fi
-  if ! list_buckets_with_user "s3api" "$1" "$2"; then
-    log 2 "error listing buckets with user '$1'"
-    return 1
-  fi
-  bucket_found=false
-  for bucket in "${bucket_array[@]}"; do
-    if [ "$bucket" == "$3" ]; then
-      log 2 "bucket '$3' shouldn't show up in user '$1' bucket list"
-      return 1
-    elif [ "$bucket" == "$4" ]; then
-      bucket_found=true
-    fi
-  done
-  if [ $bucket_found == false ]; then
-    log 2 "user-owned bucket '$4' not found in user list"
-    return 1
-  fi
-  return 0
-}

tests/drivers/list_buckets/list_buckets_s3api.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+list_and_check_buckets_omit_without_permission() {
+  if [ $# -ne 4 ]; then
+    log 2 "'list_and_check_buckets_with_user' requires username, password, non-visible bucket, visible bucket"
+    return 1
+  fi
+  if ! response=$(list_buckets_with_user "s3api" "$1" "$2" 2>&1); then
+    log 2 "error listing buckets with user '$1': $response"
+    return 1
+  fi
+  read -r -a bucket_array <<< "$response"
+
+  bucket_found=false
+  for bucket in "${bucket_array[@]}"; do
+    if [ "$bucket" == "$3" ]; then
+      log 2 "bucket '$3' shouldn't show up in user '$1' bucket list"
+      return 1
+    elif [ "$bucket" == "$4" ]; then
+      bucket_found=true
+    fi
+  done
+  if [ $bucket_found == false ]; then
+    log 2 "user-owned bucket '$4' not found in user list"
+    return 1
+  fi
+  return 0
+}

tests/drivers/put_object/put_object_rest.sh

@@ -71,6 +71,35 @@ setup_bucket_and_add_files() {
   return 0
 }
 
+setup_bucket_and_add_file_v3() {
+  if ! check_param_count_v2 "bucket name or prefix" 1 $#; then
+    return 1
+  fi
+  if ! response=$(setup_bucket_v3 "$1" 2>&1); then
+    log 2 "error setting up bucket: $response"
+    return 1
+  fi
+  bucket_name="$response"
+
+  if ! response=$(get_file_name 2>&1); then
+    log 2 "error getting file name: $response"
+    return 1
+  fi
+  file_name="$response"
+
+  if ! response=$(create_test_files "$file_name" 2>&1); then
+    log 2 " error creating test file: $response"
+    return 1
+  fi
+  if ! response=$(put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" 2>&1); then
+    log 2 "error adding file '$TEST_FILE_FOLDER/$file_name' to bucket '$bucket_name': $response"
+    return 1
+  fi
+  log 5 "file name: $file_name"
+  echo "$bucket_name $file_name"
+  return 0
+}
+
 send_openssl_go_command_chunked_no_content_length() {
   if ! check_param_count_gt "bucket name, key" 2 $#; then
     return 1

tests/drivers/rest.sh

@@ -401,6 +401,7 @@ check_for_header_key_and_value() {
   if ! check_param_count_v2 "data file, header key, header value" 3 $#; then
     return 1
   fi
+  log 5 "header data: $(cat "$1")"
   while IFS=$': \r' read -r key value; do
     local check_result=0
     value="${value%$'\r'}"

tests/drivers/string.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/logger.sh
+
+generate_random_string() {
+  if ! check_param_count_v2 "min size, max size" 2 $#; then
+    return 1
+  fi
+  local string="" range size response
+  range=$(($2 - $1 + 1))
+  size=$(($1 + RANDOM % range))
+
+  if ! response=$(get_file_name 2>&1); then
+    log 2 "error getting file name: $response"
+    return 1
+  fi
+  pipe_error_file="$response"
+
+  while [ "${#string}" -lt "$size" ]; do
+    chunk="$(
+      {
+        dd if=/dev/urandom bs=128 count=1 |
+          LC_ALL=C tr -d '\000' |
+          LC_ALL=C tr -dc 'A-Za-z0-9'
+      } 2>"$TEST_FILE_FOLDER/$pipe_error_file"
+    )"
+    status=$?
+
+    if [ $status -ne 0 ]; then
+      log 2 "error creating chunk: $(cat "$TEST_FILE_FOLDER/$pipe_error_file")"
+      return 1
+    fi
+
+    string="${string}${chunk}"
+  done
+
+  printf '%s\n' "${string:0:size}"
+  return 0
+}

tests/test_common.sh

@@ -32,7 +32,6 @@ source ./tests/commands/put_public_access_block.sh
 source ./tests/drivers/create_bucket/create_bucket_rest.sh
 source ./tests/drivers/file.sh
 source ./tests/drivers/params.sh
-source ./tests/util/util_list_buckets.sh
 source ./tests/util/util_object.sh
 source ./tests/util/util_policy.sh
 source ./tests/util/util_presigned_url.sh

tests/test_rest_bucket.sh

@@ -29,7 +29,6 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro
 source ./tests/drivers/put_bucket_tagging/put_bucket_tagging_rest.sh
 source ./tests/logger.sh
 source ./tests/setup.sh
-source ./tests/util/util_list_buckets.sh
 source ./tests/util/util_lock_config.sh
 source ./tests/util/util_public_access_block.sh
 source ./tests/util/util_rest.sh

tests/test_rest_head_object.sh

@@ -19,48 +19,112 @@ load ./bats-assert/load
 
 source ./tests/setup.sh
 source ./tests/drivers/create_bucket/create_bucket_rest.sh
+source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh
 source ./tests/drivers/get_object_attributes/get_object_attributes_rest.sh
+source ./tests/drivers/string.sh
 
 # tags: curl,HeadObject,GetObjectAttributes,ETag,x-amz-object-attributes
 @test "REST - head object" {
-  run get_file_name
+  run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME"
   assert_success
-  test_file="$output"
+  read -r bucket_name file_name <<< "$output"
 
-  run get_bucket_name "$BUCKET_ONE_NAME"
-  assert_success
-  bucket_name="$output"
-
-  run setup_bucket_and_file_v2 "$bucket_name" "$test_file"
-  assert_success
-
-  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
-  assert_success
-
-  run get_etag_rest "$bucket_name" "$test_file"
+  run get_etag_rest "$bucket_name" "$file_name"
   assert_success
   expected_etag=$output
 
-  run check_etag_attribute_rest "$bucket_name" "$test_file" "$expected_etag"
+  run check_etag_attribute_rest "$bucket_name" "$file_name" "$expected_etag"
   assert_success
 }
 
 # tags: curl,HeadObject,Content-Type
 @test "REST - HeadObject - default Content-Type is binary/octet-stream" {
-  run get_bucket_name "$BUCKET_ONE_NAME"
+  run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME"
   assert_success
-  bucket_name="$output"
+  read -r bucket_name file_name <<< "$output"
 
-  run get_file_name
-  assert_success
-  test_file="$output"
-
-  run setup_bucket_and_file_v2 "$bucket_name" "$test_file"
-  assert_success
-
-  run put_object_rest "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
-  assert_success
-
-  run head_object_check_header_key_and_value "$bucket_name" "$test_file" "Content-Type" "binary/octet-stream"
+  run head_object_check_header_key_and_value "$bucket_name" "$file_name" "Content-Type" "binary/octet-stream"
   assert_success
 }
+
+@test "REST - HeadObject - letter partNumber" {
+  run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME"
+  assert_success
+  read -r bucket_name file_name <<< "$output"
+
+  run send_rest_go_command "400" "-method" "HEAD" "-query" "partNumber=abc" "-objectKey" "$file_name" "-bucketName" "$bucket_name"
+  assert_success
+}
+
+@test "REST - HeadObject - invalid partNumber" {
+  run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME"
+  assert_success
+  read -r bucket_name file_name <<< "$output"
+
+  run send_rest_go_command "416" "-method" "HEAD" "-query" "partNumber=2" "-objectKey" "$file_name" "-bucketName" "$bucket_name"
+  assert_success
+}
+
+@test "REST - HeadObject - valid partNumbers" {
+  file_mb=8
+  run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" "$file_mb"
+  assert_success
+  read -r bucket_name test_file <<< "$output"
+  file_bytes=$((file_mb*1024*1024))
+
+  run split_file_irregular "$TEST_FILE_FOLDER/$test_file" 5242880
+  assert_success
+  read -r part_one part_two <<< "$output"
+  log 5 "part one: $part_one, part two: $part_two"
+
+  run perform_multipart_upload_rest_variable_parts "$bucket_name" "$test_file" "$part_one" "$part_two"
+  assert_success
+
+  run get_file_size "$part_one"
+  assert_success
+  part_size_one="$output"
+
+  run get_file_size "$part_one"
+  assert_success
+  part_size_two="$output"
+
+  run send_rest_go_command_callback "206" "check_header_partial_content_response" "-bucketName" "$bucket_name" "-objectKey" "$test_file" \
+    "-method" "HEAD" "-query" "partNumber=1" "--" "1" "$file_bytes" "$part_size_one"
+  assert_success
+
+  run send_rest_go_command_callback "206" "check_header_partial_content_response" "-bucketName" "$bucket_name" "-objectKey" "$test_file" \
+    "-method" "HEAD" "-query" "partNumber=2" "--" "2" "$file_bytes" "$part_size_two"
+  assert_success
+}
+
+@test "REST - HeadObject - invalid request header type" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/2075"
+  fi
+  run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME"
+  assert_success
+  read -r bucket_name file_name <<< "$output"
+
+  run send_rest_go_command "400" "-bucketName" "$bucket_name" "-objectKey" "$file_name" \
+    "-method" "HEAD" "-query" "response-invalid=invalid"
+  assert_success
+}
+
+@test "REST - HeadObject - response queries" {
+  run setup_bucket_and_add_file_v3 "$BUCKET_ONE_NAME"
+  assert_success
+  read -r bucket_name file_name <<< "$output"
+
+  query_suffixes=("cache-control" "content-disposition" "content-encoding" "content-language" "content-type" "expires")
+
+  for suffix in "${query_suffixes[@]}"; do
+    run generate_random_string 20 40
+    assert_success
+    value=$output
+    log 5 "value: $value"
+
+    run send_rest_go_command_callback "200" "check_for_header_key_and_value" "-bucketName" "$bucket_name" "-objectKey" "$file_name" \
+      "-method" "HEAD" "-query" "response-$suffix=$value" "--" "$suffix" "$value"
+    assert_success
+  done
+}

tests/test_s3.sh

@@ -18,6 +18,7 @@ load ./bats-support/load
 load ./bats-assert/load
 
 source ./tests/drivers/create_bucket/create_bucket_rest.sh
+source ./tests/drivers/list_buckets/list_buckets.sh
 source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
 source ./tests/drivers/file.sh
 source ./tests/test_common.sh

tests/test_s3api_bucket.sh

@@ -52,6 +52,7 @@ source ./tests/drivers/get_bucket_tagging/get_bucket_tagging.sh
 source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh
 source ./tests/drivers/head_bucket/head_bucket_rest.sh
 source ./tests/drivers/head_bucket/head_bucket_s3api.sh
+source ./tests/drivers/list_buckets/list_buckets.sh
 source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
 source ./tests/util/util_lock_config.sh
 source ./tests/util/util_object.sh

tests/test_s3cmd.sh

@@ -24,6 +24,7 @@ source ./tests/commands/delete_bucket_policy.sh
 source ./tests/commands/get_bucket_policy.sh
 source ./tests/commands/put_bucket_policy.sh
 source ./tests/drivers/get_bucket_location/get_bucket_location.sh
+source ./tests/drivers/list_buckets/list_buckets.sh
 source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
 source ./tests/util/util_object.sh
 

tests/test_user_aws.sh

@@ -20,6 +20,8 @@ load ./bats-assert/load
 source ./tests/test_user_common.sh
 source ./tests/commands/get_object.sh
 source ./tests/commands/put_object.sh
+source ./tests/drivers/list_buckets/list_buckets.sh
+source ./tests/drivers/list_buckets/list_buckets_s3api.sh
 source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
 source ./tests/drivers/file.sh
 source ./tests/drivers/user.sh

tests/test_user_common.sh

@@ -16,7 +16,6 @@
 
 source ./tests/setup.sh
 source ./tests/drivers/create_bucket/create_bucket_rest.sh
-source ./tests/util/util_list_buckets.sh
 source ./tests/util/util_object.sh
 source ./tests/util/util_users.sh
 source ./tests/commands/list_buckets.sh

tests/test_user_s3cmd.sh

@@ -17,6 +17,8 @@
 load ./bats-support/load
 load ./bats-assert/load
 
+source ./tests/drivers/list_buckets/list_buckets.sh
+source ./tests/drivers/list_buckets/list_buckets_s3api.sh
 source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
 source ./tests/drivers/user.sh
 source ./tests/test_user_common.sh