test: chunked uploads w/o trailers - first tests

tests/commands/get_bucket_ownership_controls.sh

@@ -26,6 +26,7 @@ get_bucket_ownership_controls() {
     return 1
   fi
 
+  raw_bucket_ownership_controls=""
   if ! raw_bucket_ownership_controls=$(send_command aws --no-verify-ssl s3api get-bucket-ownership-controls --bucket "$1" 2>&1); then
     log 2 "error getting bucket ownership controls: $raw_bucket_ownership_controls"
     return 1

tests/logger.sh

@@ -16,15 +16,13 @@
 
 # levels:  1 - crit, 2 - err, 3 - warn, 4 - info, 5 - debug, 6 - trace
 
-export LOG_LEVEL_INT=4
-
 log() {
   if [[ $# -ne 2 ]]; then
     echo "log function requires level, message"
     return 1
   fi
   # shellcheck disable=SC2153
-  if [[ $1 -gt $LOG_LEVEL_INT ]]; then
+  if [[ $1 -gt ${LOG_LEVEL_INT:=4} ]]; then
     return 0
   fi
   log_level=""

tests/rest_scripts/abort_multipart_upload.sh

@@ -37,6 +37,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id"

tests/rest_scripts/complete_multipart_upload.sh

@@ -41,6 +41,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id"

tests/rest_scripts/copy_object.sh

@@ -38,6 +38,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-copy-source;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key")

tests/rest_scripts/create_multipart_upload.sh

@@ -39,6 +39,7 @@ UNSIGNED-PAYLOAD"
 
 canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploads="

tests/rest_scripts/delete_object_tagging.sh

@@ -35,6 +35,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?tagging"

tests/rest_scripts/delete_objects.sh

@@ -49,6 +49,7 @@ fi
 canonical_request+="host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name?delete")

tests/rest_scripts/get_bucket_acl.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?acl="

tests/rest_scripts/get_bucket_ownership_controls.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?ownershipControls="

tests/rest_scripts/get_bucket_policy.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?policy="

tests/rest_scripts/get_bucket_tagging.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?tagging="

tests/rest_scripts/get_bucket_versioning.sh

@@ -32,6 +32,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?versioning="

tests/rest_scripts/get_object.sh

@@ -36,6 +36,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key"

tests/rest_scripts/get_object_attributes.sh

@@ -41,6 +41,7 @@ x-amz-object-attributes:$attributes
 host;x-amz-content-sha256;x-amz-date;x-amz-object-attributes
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key?attributes="

tests/rest_scripts/get_object_legal_hold.sh

@@ -35,6 +35,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key?legal-hold="

tests/rest_scripts/get_public_access_block.sh

@@ -33,6 +33,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock="

tests/rest_scripts/head_object.sh

@@ -36,6 +36,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ksI -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key"

tests/rest_scripts/list_buckets.sh

@@ -28,6 +28,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host"

tests/rest_scripts/list_multipart_uploads.sh

@@ -36,6 +36,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name/$key?uploads="

tests/rest_scripts/list_object_versions.sh

@@ -34,6 +34,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name?versions"

tests/rest_scripts/list_objects.sh

@@ -57,6 +57,8 @@ x-amz-date:$current_date_time
 
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
+
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"")

tests/rest_scripts/list_parts.sh

@@ -38,6 +38,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name/$key?uploadId=$upload_id"

tests/rest_scripts/put_bucket_acl.sh

@@ -54,6 +54,7 @@ fi
 canonical_request+="x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl=")

tests/rest_scripts/put_bucket_ownership_controls.sh

@@ -45,6 +45,7 @@ x-amz-date:$current_date_time
 content-md5;host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?ownershipControls="

tests/rest_scripts/put_bucket_policy.sh

@@ -38,6 +38,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?policy="

tests/rest_scripts/put_bucket_tagging.sh

@@ -50,6 +50,7 @@ x-amz-date:$current_date_time
 content-md5;host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?tagging="

tests/rest_scripts/put_bucket_versioning.sh

@@ -41,6 +41,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?versioning="

tests/rest_scripts/put_object.sh

@@ -54,6 +54,7 @@ host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 fi
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key")

tests/rest_scripts/put_object_openssl_chunked_example.sh

@@ -0,0 +1,351 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+source ./tests/util/util_file.sh
+
+# Fields
+
+load_parameters() {
+  test_mode=${TEST_MODE:=true}
+  # shellcheck disable=SC2034
+  command_file="${COMMAND_FILE:=command.txt}"
+
+  readonly signature_no_data="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+  if [ "$test_mode" == "true" ]; then
+    current_date_time="20130524T000000Z"
+    year_month_day="20130524"
+    bucket_name="examplebucket"
+    key="chunkObject.txt"
+    aws_access_key_id="AKIAIOSFODNN7EXAMPLE"
+    # shellcheck disable=SC2034
+    aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+    aws_region="us-east-1"
+    host="s3.amazonaws.com"
+    dd if=/dev/zero bs=1 count=66560 | tr '\0' 'a' > "as.txt"
+    data_file="as.txt"
+    chunk_size=65536
+  else
+    current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+    year_month_day=$(echo "$current_date_time" | cut -c1-8)
+    # shellcheck disable=SC2153
+    bucket_name="$BUCKET_NAME"
+    key="$OBJECT_KEY"
+    # shellcheck disable=SC2153
+    data_file="$DATA_FILE"
+    chunk_size="${CHUNK_SIZE:=65536}"
+  fi
+
+  readonly initial_sts_data="AWS4-HMAC-SHA256-PAYLOAD
+$current_date_time
+$year_month_day/$aws_region/s3/aws4_request"
+
+  if [ "$test_mode" == "true" ]; then
+    declare_test_expected_vals
+  fi
+}
+
+declare_test_expected_vals() {
+  readonly expected_sts_data="AWS4-HMAC-SHA256
+20130524T000000Z
+20130524/us-east-1/s3/aws4_request
+cee3fed04b70f867d036f722359b0b1f2f0e5dc0efadbc082b76c4c60e316455"
+
+  readonly expected_sts_chunk_one="AWS4-HMAC-SHA256-PAYLOAD
+20130524T000000Z
+20130524/us-east-1/s3/aws4_request
+4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+bf718b6f653bebc184e1479f1935b8da974d701b893afcf49e701f3e2f9f9c5a"
+
+  readonly expected_sts_chunk_two="AWS4-HMAC-SHA256-PAYLOAD
+20130524T000000Z
+20130524/us-east-1/s3/aws4_request
+ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a"
+
+  readonly expected_sts_chunk_three="AWS4-HMAC-SHA256-PAYLOAD
+20130524T000000Z
+20130524/us-east-1/s3/aws4_request
+0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+  readonly expected_canonical_request="PUT
+/examplebucket/chunkObject.txt
+
+content-encoding:aws-chunked
+content-length:66824
+host:s3.amazonaws.com
+x-amz-content-sha256:STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+x-amz-date:20130524T000000Z
+x-amz-decoded-content-length:66560
+x-amz-storage-class:REDUCED_REDUNDANCY
+
+content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class
+STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+
+  readonly expected_command="PUT /examplebucket/chunkObject.txt HTTP/1.1
+Host: s3.amazonaws.com
+x-amz-date: 20130524T000000Z
+x-amz-storage-class: REDUCED_REDUNDANCY
+Authorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9
+x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+Content-Encoding: aws-chunked
+x-amz-decoded-content-length: 66560
+Content-Length: 66824
+
+"
+}
+
+get_file_size_and_content_length() {
+  if ! file_size=$(get_file_size "$data_file"); then
+    log_rest 2 "error getting file size: $file_size"
+    return 1
+  fi
+  if [ "$test_mode" == "true" ] && [ "$file_size" != 66560 ]; then
+    log_rest 2 "file size mismatch ($file_size)"
+    return 1
+  fi
+
+  get_chunk_sizes
+  content_length=$((length+file_size))
+  if [ "$test_mode" == "true" ] && [ "$content_length" != 66824 ]; then
+    log_rest 2 "content length mismatch ($content_length)"
+    return 1
+  fi
+}
+
+get_chunk_sizes() {
+  chunk_sizes=()
+  for ((remaining=file_size; 0<remaining; remaining-=chunk_size)); do
+    if ((chunk_size<=remaining)); then
+      next_chunk_size=$chunk_size
+    else
+      next_chunk_size=$remaining
+    fi
+    chunk_sizes+=("$next_chunk_size")
+  done
+  chunk_sizes+=("0")
+  for size in "${chunk_sizes[@]}"; do
+    hex_size=$(printf "%x\n" "$size")
+    length=$((length+${#hex_size}+85))
+  done
+}
+
+get_first_signature() {
+  if [ -n "$FIRST_SIGNATURE" ]; then
+    first_signature="$FIRST_SIGNATURE"
+    return 0
+  fi
+
+  canonical_request="PUT
+/$bucket_name/$key
+
+content-encoding:aws-chunked
+content-length:$content_length
+host:$host
+x-amz-content-sha256:STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+x-amz-date:$current_date_time
+x-amz-decoded-content-length:$file_size
+x-amz-storage-class:REDUCED_REDUNDANCY
+
+content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class
+STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+
+  if [ "$test_mode" == "true" ]; then
+    if [ "$expected_canonical_request" != "$canonical_request" ]; then
+      log_rest 2 "canonical request mismatch ($canonical_request)"
+      exit 1
+    fi
+  fi
+
+  # shellcheck disable=SC2119
+  create_canonical_hash_sts_and_signature
+  # shellcheck disable=SC2154
+  first_signature="$signature"
+
+  if [ "$test_mode" == "true" ]; then
+    # shellcheck disable=SC2154
+    if [ "$sts_data" != "$expected_sts_data" ]; then
+      log_rest 2 "first STS data mismatch ($sts_data)"
+      return 1
+    fi
+    if [ "$first_signature" != "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9" ]; then
+      log_rest 2 "Mismatched first signature ($first_signature)"
+      return 1
+    fi
+  fi
+}
+
+create_chunk() {
+  if [ $# -ne 4 ]; then
+    log_rest 2 "'generate_chunk_signature' requires previous signature, file, offset, bytes"
+    return 1
+  fi
+  chunk_sts_data="$initial_sts_data
+$1
+$signature_no_data
+"
+  if [ "$4" -ne 0 ]; then
+    if ! data=$(dd if="$2" of="$2.tmp" bs=1 skip="$3" count="$4" 2>&1); then
+      log_rest 2 "error retrieving data: $data"
+      return 1
+    fi
+    payload_hash="$(sha256sum "$2.tmp" | awk '{print $1}')"
+  else
+    payload_hash="$signature_no_data"
+  fi
+  chunk_sts_data+="$payload_hash"
+  create_canonical_hash_sts_and_signature "$chunk_sts_data"
+  chunk="$(printf "%x" "$4");chunk-signature=$signature"
+  if [ "$4" -gt 0 ]; then
+    chunk+="
+$(cat "$2.tmp")"
+  fi
+}
+
+build_chunks() {
+  if [ $# -ne 1 ]; then
+    log_rest 2 "'add_chunks' requires first signature"
+    return 1
+  fi
+
+  last_signature="$1"
+  idx=0
+  offset=0
+  chunks=""
+  for chunk_size in "${chunk_sizes[@]}"; do
+    if ! build_chunk; then
+      log 2 "error building chunk"
+      return 1
+    fi
+    if [ "$test_mode" == "true" ]; then
+      check_chunks_and_signatures_in_test_mode $idx
+    fi
+    ((idx++))
+  done
+}
+
+build_chunk() {
+  if [ "$chunk_size" == 0 ]; then
+    if ! create_chunk "$last_signature" "$data_file" 0 0; then
+      log_rest 2 "error creating chunk $idx"
+      return 1
+    fi
+  else
+    if ! create_chunk "$last_signature" "$data_file" "$offset" "${chunk_sizes[$idx]}"; then
+      log_rest 2 "error creating chunk $idx"
+      return 1
+    fi
+    offset=$((offset+chunk_size))
+    last_signature="$signature"
+  fi
+  chunks+="$chunk
+"
+}
+
+check_chunks_and_signatures_in_test_mode() {
+  if [ $# -ne 1 ]; then
+    log_rest 2 "'check_chunks_and_signatures_in_test_mode' requires chunk number"
+    return 1
+  fi
+  case "$1" in
+    0)
+      if [ "$chunk_sts_data" != "$expected_sts_chunk_one" ]; then
+        log_rest 2 "first chunk STS mismatch ($chunk_sts_data)"
+        return 1
+      fi
+      if [ "$signature" != "ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648" ]; then
+        log_rest 2 "first chunk signature mismatch ($signature)"
+        return 1
+      fi
+      ;;
+    1)
+      if [ "$chunk_sts_data" != "$expected_sts_chunk_two" ]; then
+        log_rest 2 "second chunk STS mismatch ($chunk_sts_data)"
+        return 1
+      fi
+      if [ "$signature" != "0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497" ]; then
+        log_rest 2 "second chunk signature mismatch ($signature)"
+        return 1
+      fi
+      ;;
+    2)
+      if [ "$chunk_sts_data" != "$expected_sts_chunk_three" ]; then
+        log_rest 2 "final chunk STS mismatch ($chunk_sts_data)"
+        return 1
+      fi
+      if [ "$signature" != "b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9" ]; then
+        log_rest 2 "final chunk signature mismatch ($signature)"
+        return 1
+      fi
+      ;;
+  esac
+}
+
+build_command() {
+  command="PUT /$bucket_name/$key HTTP/1.1
+Host: $host
+x-amz-date: $current_date_time
+x-amz-storage-class: REDUCED_REDUNDANCY
+Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=$first_signature
+x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD
+Content-Encoding: aws-chunked
+x-amz-decoded-content-length: $file_size
+Content-Length: $content_length
+
+"
+
+if [ "$test_mode" == "true" ] && [ "$command" != "$expected_command" ]; then
+  log_rest 2 "command mismatch ($command)"
+  return 1
+fi
+
+  chunks+="
+"
+  command+="$chunks"
+  command="${command//$'\n'/$'\r\n'}"
+  echo -n "$command" > "$COMMAND_FILE"
+}
+
+load_parameters
+
+if ! get_file_size_and_content_length; then
+  log 2 "error getting file size and content length"
+  exit 1
+fi
+
+if ! get_first_signature; then
+  log 2 "error getting first signature"
+  exit 1
+fi
+
+if ! build_chunks "$first_signature"; then
+  log 2 "error building chunks"
+  exit 1
+fi
+
+if ! build_command; then
+  log 2 "error building command"
+  exit 1
+fi
+if [ "$test_mode" == "true" ]; then
+  echo "TEST PASS"
+fi
+exit 0

tests/rest_scripts/put_object_retention.sh

@@ -48,6 +48,7 @@ x-amz-date:$current_date_time
 content-md5;host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key?retention="

tests/rest_scripts/put_public_access_block.sh

@@ -46,6 +46,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock="

tests/rest_scripts/rest.sh

@@ -15,6 +15,7 @@
 # under the License.
 
 source ./tests/commands/command.sh
+source ./tests/logger.sh
 
 # shellcheck disable=SC2153,SC2034
 aws_access_key_id="$AWS_ACCESS_KEY_ID"
@@ -48,10 +49,14 @@ create_canonical_hash_sts_and_signature() {
   # shellcheck disable=SC2154
   year_month_day="$(echo "$current_date_time" | cut -c1-8)"
 
-  sts_data="AWS4-HMAC-SHA256
+  if [ $# -eq 0 ]; then
+    sts_data="AWS4-HMAC-SHA256
 $current_date_time
 $year_month_day/$aws_region/s3/aws4_request
 $canonical_request_hash"
+  else
+    sts_data="$1"
+  fi
 
   date_key=$(echo -n "$year_month_day" | openssl dgst -sha256 -mac HMAC -macopt key:"AWS4${aws_secret_access_key}" | awk '{print $2}')
   date_region_key=$(echo -n "$aws_region" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"$date_key" | awk '{print $2}')
@@ -62,18 +67,36 @@ $canonical_request_hash"
                    -mac HMAC \
                    -macopt hexkey:"$signing_key" | awk '{print $2}')
 
-  curl_command=()
-  add_command_recording_if_enabled
+  if [ $# -eq 0 ]; then
+    curl_command=()
+    add_command_recording_if_enabled
+  fi
 }
 
 add_parameter() {
-  if [ "$#" -ne 2 ]; then
+  if [ "$#" -lt 2 ]; then
     return
   fi
+  if [ "$3" != "" ]; then
+    divider="$3"
+  else
+    divider="&"
+  fi
   current_string="$1"
   if [ "$current_string" != "" ]; then
-    current_string+="&"
+    current_string+="$divider"
   fi
   current_string+="$2"
   echo "$current_string"
 }
+
+log_rest() {
+  if [ $# -ne 2 ]; then
+    return 1
+  fi
+  if [ "$BATS_TEST_NAME" != "" ]; then
+    log "$1" "$2"
+  else
+    echo "$2"
+  fi
+}

tests/rest_scripts/upload_part.sh

@@ -44,6 +44,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-date
 $payload_hash"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -isk -w "\"%{http_code}\"" "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\""

tests/rest_scripts/upload_part_copy.sh

@@ -43,6 +43,7 @@ x-amz-date:$current_date_time
 host;x-amz-content-sha256;x-amz-copy-source;x-amz-date
 UNSIGNED-PAYLOAD"
 
+# shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\""

tests/run.sh

@@ -141,9 +141,11 @@ run_suite() {
       ;;
     rest)
       echo "Running rest tests ..."
-      if "$HOME"/bin/bats ./tests/test_rest.sh; then
-        "$HOME"/bin/bats ./tests/test_rest_acl.sh || exit_code=$?
-      else
+      if ! "$HOME"/bin/bats ./tests/test_rest.sh; then
+        exit_code=1
+      elif ! "$HOME"/bin/bats ./tests/test_rest_acl.sh; then
+        exit_code=1
+      elif ! "$HOME"/bin/bats ./tests/test_rest_chunked.sh; then
         exit_code=1
       fi
       ;;

tests/test_rest_chunked.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+load ./bats-support/load
+load ./bats-assert/load
+
+source ./tests/logger.sh
+source ./tests/setup.sh
+source ./tests/util/util_bucket.sh
+source ./tests/util/util_chunked_upload.sh
+source ./tests/util/util_file.sh
+source ./tests/util/util_head_object.sh
+
+@test "REST - chunked upload, no content length" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1043"
+  fi
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test-file"
+  run create_test_files "$test_file"
+  assert_success
+
+  run attempt_seed_signature_without_content_length "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
+  assert_success
+}
+
+@test "REST - chunked upload, signature error" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1056 - gibberish at end"
+  fi
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test-file"
+  run create_test_file "$test_file" 8192
+  assert_success
+
+  run attempt_chunked_upload_with_bad_first_signature "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}

tests/util/util_acl.sh

@@ -401,10 +401,12 @@ check_ownership_rule_and_reset_acl() {
     return 1
   fi
   # shellcheck disable=SC2154
+  log 5 "ownership controls: $bucket_ownership_controls"
   if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then
     log 2 "error getting object ownership rule: $object_ownership_rule"
     return 1
   fi
+  log 5 "ownership rule: $object_ownership_rule"
   if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then
     log 2 "error resetting bucket ACL"
     return 1

tests/util/util_bucket.sh

@@ -55,11 +55,8 @@ clear_bucket_s3api() {
     return 1
   fi
 
-  if [[ $LOG_LEVEL_INT -ge 5 ]]; then
-    if ! log_bucket_policy "$1"; then
-      log 2 "error logging bucket policy"
-      return 1
-    fi
+  if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$1"; then
+    log 3 "error logging bucket policy"
   fi
 
   if ! check_object_lock_config "$1"; then

tests/util/util_chunked_upload.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+attempt_seed_signature_without_content_length() {
+  if [ "$#" -ne 3 ]; then
+    log 2 "'attempt_seed_signature_without_content_length' requires bucket name, key, data file"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" CONTENT_ENCODING="aws-chunked" BUCKET_NAME="$1" OBJECT_KEY="$2" DATA_FILE="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then
+    log 2 "error putting object: $result"
+    return 1
+  fi
+  if [ "$result" != 411 ]; then
+    log 2 "expected '411', actual '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+    return 1
+  fi
+  return 0
+}
+
+attempt_chunked_upload_with_bad_first_signature() {
+  if [ $# -ne 3 ]; then
+    log 2 "'attempt_chunked_upload_with_bad_first_signature' requires data file, bucket name, key"
+    return 1
+  fi
+  if ! result=$(AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
+         AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
+         AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \
+         DATA_FILE="$1" \
+         BUCKET_NAME="$2" \
+         OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" FIRST_SIGNATURE="xxxxxxxx" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then
+    log 2 "error creating command: $result"
+    return 1
+  fi
+
+  host="${AWS_ENDPOINT_URL#http*://}"
+  if [ "$host" == "s3.amazonaws.com" ]; then
+    host+=":443"
+  fi
+  if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then
+    log 2 "error sending openssl command: $result"
+    return 1
+  fi
+  response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
+  log 5 "response code: $response_code"
+  if [ "$response_code" != "403" ]; then
+    log 2 "expected code '403', was '$response_code'"
+    return 1
+  fi
+  response_data="$(echo "$result" | grep "<")"
+  log 5 "response data: $response_data"
+  log 5 "END"
+  if ! check_xml_element <(echo "$response_data") "SignatureDoesNotMatch" "Error" "Code"; then
+    log 2 "error checking XML element"
+    return 1
+  fi
+  return 0
+}

tests/util/util_file.sh

@@ -46,7 +46,7 @@ create_test_file() {
       return 1
     fi
   fi
-  if ! error=$(touch "$TEST_FILE_FOLDER/$1"); then
+  if ! error=$(touch "$TEST_FILE_FOLDER/$1" 2>&1); then
     log 2 "error creating new file: $error"
     return 1
   fi
@@ -102,6 +102,26 @@ delete_test_files() {
   return 0
 }
 
+get_file_size() {
+  if [ $# -ne 1 ]; then
+    log 2 "'get_file_size' requires file location"
+    return 1
+  fi
+  local file_size=""
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    if ! file_size=$(stat -f %z "$1" 2>&1); then
+      log 2 "error getting file size: $file_size"
+      return 1
+    fi
+  else
+    if ! file_size=$(stat -c %s "$1" 2>&1); then
+      log 2 "error getting file size: $file_size"
+      return 1
+    fi
+  fi
+  echo "$file_size"
+}
+
 # split file into pieces to test multipart upload
 # param: file location
 # return 0 for success, 1 for error

tests/util/util_list_parts.sh

@@ -1,5 +1,27 @@
 #!/usr/bin/env bash
 
+check_part_list_rest_etags() {
+  if ! etags=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/parts.txt" | tr '\n' ' ' 2>&1); then
+    log 2 "error retrieving etags: $etags"
+    return 1
+  fi
+  read -ra etags_array <<< "$etags"
+  idx=0
+  if [ $# -ne ${#etags_array[@]} ]; then
+    log 2 "expected tag size of '$#', actual ${#etags_array[@]}"
+    return 1
+  fi
+  while [ $# -gt 0 ]; do
+    if [ "$1" != "${etags_array[$idx]}" ]; then
+      log 2 "etag mismatch (expected '$1', actual ${etags_array[$idx]})"
+      return 1
+    fi
+    ((idx++))
+    shift
+  done
+  return 0
+}
+
 check_part_list_rest() {
   if [ $# -lt 4 ]; then
     log 2 "'check_part_list_rest' requires bucket, file name, upload ID, expected count, etags"
@@ -33,21 +55,10 @@ check_part_list_rest() {
   if [ "$4" == 0 ]; then
     return 0
   fi
-  if ! etags=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/parts.txt" | tr '\n' ' ' 2>&1); then
-    log 2 "error retrieving etags: $etags"
+  if ! check_part_list_rest_etags "${@:5}"; then
+    log 2 "error checking etags"
     return 1
   fi
-  read -ra etags_array <<< "$etags"
-  shift 4
-  idx=0
-  while [ $# -gt 0 ]; do
-    if [ "$1" != "${etags_array[$idx]}" ]; then
-      log 2 "etag mismatch (expected '$1', actual ${etags_array[$idx]})"
-      return 1
-    fi
-    ((idx++))
-    shift
-  done
   return 0
 }