test: chunked upload - invalid checksums, retention, legal hold tests

tests/rest_scripts/calculate_checksum.py

@@ -0,0 +1,21 @@
+import sys
+from awscrt import checksums
+
+def get_call(checksum_type):
+    if checksum_type == "crc32c":
+        return checksums.crc32c
+    elif checksum_type == "crc64nvme":
+        return checksums.crc64nvme
+    sys.stderr.write("unrecognized checksum type " + checksum_type)
+    sys.exit(1)
+
+def main():
+    if len(sys.argv) != 3:
+        sys.stderr.write('Checksum type, data file path required')
+        sys.exit(1)
+    with open(sys.argv[2], 'rb') as f:
+        function = get_call(sys.argv[1])
+        print(function(f.read()))
+
+if __name__ == "__main__":
+    main()

tests/rest_scripts/calculate_checksum.sh

@@ -16,29 +16,59 @@
 
 source ./tests/rest_scripts/rest.sh
 
-if ! DEACTIVATE=false source ./tests/rest_scripts/init_python_env.sh; then
-  log_rest 2 "error initializing python environment"
-  exit 1
-fi
-if ! checksum_decimal=$(python3 -c "
-import sys
-from awscrt import checksums
+calculate_checksum_python() {
+  if [ "$#" -ne 2 ]; then
+    log 2 "'calculate_checksum_python' requires checksum type, data file"
+    return 1
+  fi
+  if ! DEACTIVATE=false source ./tests/rest_scripts/init_python_env.sh; then
+    log_rest 2 "error initializing python environment"
+    return 1
+  fi
+  if ! checksum_decimal=$(python3 ./tests/rest_scripts/calculate_checksum.py "$1" "$2" 2>&1); then
+    log_rest 2 "error calculating checksum: $checksum_decimal"
+    return 1
+  fi
+  log 5 "decimal checksum: $checksum_decimal"
+  if ! deactivate 1>/dev/null; then
+    log_rest 2 "error deactivating virtual environment"
+    return 1
+  fi
+  if [ "$CHECKSUM_TYPE" == "crc64nvme" ]; then
+    hex_format="%016x"
+  else
+    hex_format="%08x"
+  fi
+  # shellcheck disable=SC2059
+  checksum=$(printf "$hex_format" "$checksum_decimal" | xxd -r -p | base64)
+  echo "$checksum"
+}
 
-with open(sys.argv[1], 'rb') as f:
-  print(checksums.${CHECKSUM_TYPE}(f.read()))" "$DATA_FILE" 2>&1); then
-  log_rest 2 "error calculating checksum: $checksum_decimal"
+case "$CHECKSUM_TYPE" in
+"crc32c")
+  if ! checksum=$(calculate_checksum_python "crc32c" "$DATA_FILE" 2>&1); then
+    log_rest 2 "error getting checksum: $checksum"
+    exit 1
+  fi
+  ;;
+"crc64nvme")
+  if ! checksum=$(calculate_checksum_python "crc64nvme" "$DATA_FILE" 2>&1); then
+    log 2 "error calculating checksum: $checksum"
+    exit 1
+  fi
+  ;;
+"sha256")
+  checksum="$(sha256sum "$DATA_FILE" | awk '{print $1}' | xxd -r -p | base64)"
+  ;;
+"sha1")
+  checksum="$(sha1sum "$DATA_FILE" | awk '{print $1}' | xxd -r -p | base64)"
+  ;;
+"crc32")
+  checksum="$(gzip -c -1 "$DATA_FILE" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
+  ;;
+*)
+  log_rest 2 "invalid checksum type: '$CHECKSUM_TYPE'"
   exit 1
-fi
-log 5 "decimal checksum: $checksum_decimal"
-if ! deactivate 1>/dev/null; then
-  log_rest 2 "error deactivating virtual environment"
-  exit 1
-fi
-if [ "$CHECKSUM_TYPE" == "crc64nvme" ]; then
-  hex_format="%016x"
-else
-  hex_format="%08x"
-fi
-# shellcheck disable=SC2059
-checksum_hash=$(printf "$hex_format" "$checksum_decimal" | xxd -r -p | base64)
-echo "$checksum_hash"
+  ;;
+esac
+echo "$checksum"

tests/rest_scripts/put_object.sh

@@ -49,35 +49,12 @@ if [ -n "$expires" ]; then
   cr_data+=("expires:$expires")
 fi
 cr_data+=("host:$host")
-if [ "$checksum_type" == "sha256" ]; then
-  if [ -z "$checksum_hash" ]; then
-    checksum_hash="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
-  fi
-  cr_data+=("x-amz-checksum-sha256:$checksum_hash")
-elif [ "$checksum_type" == "sha1" ]; then
-  if [ -z "$checksum_hash" ]; then
-    checksum_hash="$(sha1sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
-  fi
-  cr_data+=("x-amz-checksum-sha1:$checksum_hash")
-elif [ "$checksum_type" == "crc32" ]; then
-  if [ -z "$checksum_hash" ]; then
-    checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
-  fi
-  cr_data+=("x-amz-checksum-crc32:$checksum_hash")
-elif [ "$checksum_type" == "crc64nvme" ]; then
-  if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc64nvme" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
-    log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
+if [ "$checksum_type" != "" ]; then
+  if [ "$checksum_hash" == "" ] && ! checksum_hash=$(DATA_FILE="$data_file" CHECKSUM_TYPE="$checksum_type" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
+    log_rest 2 "error calculating checksum hash"
     exit 1
   fi
-  cr_data+=("x-amz-checksum-crc64nvme:$checksum_hash")
-elif [ "$checksum_type" == "crc32c" ]; then
-  if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc32c" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
-    log_rest 2 "error calculating crc32c checksum: $checksum_hash"
-    exit 1
-  fi
-  cr_data+=("x-amz-checksum-crc32c:$checksum_hash")
-elif [ "$checksum_type" != "" ]; then
-  cr_data+=("x-amz-checksum-$checksum_type:$checksum_hash")
+  cr_data+=("x-amz-checksum-${checksum_type}:$checksum_hash")
 fi
 cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
 build_canonical_request "${cr_data[@]}"

tests/rest_scripts/put_object_legal_hold.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+# shellcheck disable=SC2153
+key="$OBJECT_KEY"
+# shellcheck disable=SC2153
+status="$STATUS"
+# shellcheck disable=SC2153
+omit_payload="${OMIT_PAYLOAD:=false}"
+
+if [ "$omit_payload" == "false" ]; then
+  payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
+<LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
+   <Status>$status</Status>
+</LegalHold>"
+fi
+
+payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
+content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64)
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="PUT
+/$bucket_name/$key
+legal-hold=
+content-md5:$content_md5
+host:$host
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+content-md5;host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+# shellcheck disable=SC2119
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key?legal-hold="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"Content-MD5: $content_md5\""
+-H "\"x-amz-content-sha256: $payload_hash\""
+-H "\"x-amz-date: $current_date_time\"")
+if [ "$omit_payload" == "false" ]; then
+  curl_command+=(-d "\"${payload//\"/\\\"}\"")
+fi
+curl_command+=(-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/put_object_openssl_chunked_example.sh

@@ -23,6 +23,7 @@ load_parameters() {
   test_mode=${TEST_MODE:=true}
   # shellcheck disable=SC2034
   command_file="${COMMAND_FILE:=command.txt}"
+  no_content_length="${NO_CONTENT_LENGTH:=false}"
 
   readonly signature_no_data="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 
@@ -163,8 +164,15 @@ content-length:$content_length
 host:$host
 x-amz-content-sha256:STREAMING-AWS4-HMAC-SHA256-PAYLOAD
 x-amz-date:$current_date_time
-x-amz-decoded-content-length:$file_size
-x-amz-storage-class:REDUCED_REDUNDANCY
+"
+if [ "$no_content_length" == "false" ]; then
+  canonical_request+="x-amz-decoded-content-length:$file_size
+"
+else
+  canonical_request+="x-amz-decoded-content-length:
+"
+fi
+canonical_request+="x-amz-storage-class:REDUCED_REDUNDANCY
 
 content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class
 STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
@@ -322,8 +330,12 @@ x-amz-storage-class: REDUCED_REDUNDANCY\r
 Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=$first_signature\r
 x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD\r
 Content-Encoding: aws-chunked\r
-x-amz-decoded-content-length: $file_size\r
-Content-Length: $content_length\r
+"
+if [ "$no_content_length" == "false" ]; then
+  command+="x-amz-decoded-content-length: $file_size\r
+"
+fi
+command+="Content-Length: $content_length\r
 \r\n"
 
 if [ "$test_mode" == "true" ] && [ "$command" != "$expected_command" ]; then

tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh

@@ -53,13 +53,15 @@ load_parameters() {
     final_signature="$FINAL_SIGNATURE"
     # shellcheck disable=SC2153
     trailer="$TRAILER"
+    # shellcheck disable=SC2153
+    checksum="$CHECKSUM"
   fi
 
   readonly initial_sts_data="AWS4-HMAC-SHA256-PAYLOAD
 $current_date_time
 $year_month_day/$aws_region/s3/aws4_request"
 
-  readonly initial_trailer_sts_data="AWS4-HMAC-SHA256-TRAILER
+  readonly trailer_sts_data="AWS4-HMAC-SHA256-TRAILER
 $current_date_time
 $year_month_day/$aws_region/s3/aws4_request"
 
@@ -95,7 +97,7 @@ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 
-  readonly expected_sts_chunk_final="AWS4-HMAC-SHA256-PAYLOAD
+  readonly expected_sts_chunk_final="AWS4-HMAC-SHA256-TRAILER
 20130524T000000Z
 20130524/us-east-1/s3/aws4_request
 2ca2aba2005185cf7159c6277faf83795951dd77a3a99e6e65d5c9f85863f992
@@ -124,7 +126,7 @@ x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER\r
 Content-Encoding: aws-chunked\r
 x-amz-decoded-content-length: 66560\r
 x-amz-trailer: x-amz-checksum-crc32c\r
-Content-Length: 66824\r
+Content-Length: 66946\r
 \r\n"
 }
 
@@ -141,42 +143,26 @@ get_file_size_and_content_length() {
   get_chunk_sizes
   log_rest 5 "signature string length: ${#signature_string}"
   content_length=$((length+file_size+${#signature_string}+92))
-  if [ "$test_mode" == "true" ] && [ "$content_length" != 66824 ]; then
+  if [ "$test_mode" == "true" ] && [ "$content_length" != 66946 ]; then
     log_rest 2 "content length mismatch ($content_length)"
     return 1
   fi
 }
 
 calculate_checksum() {
-  case "$TRAILER" in
-  "x-amz-checksum-crc32c")
-    if ! checksum=$(DATA_FILE=$data_file TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc32c" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
+  checksum_type="${trailer/x-amz-checksum-/}"
+  log_rest 5 "checksum type: $checksum_type"
+  if [ "$CHECKSUM" == "" ]; then
+    if ! checksum=$(DATA_FILE="$data_file" CHECKSUM_TYPE="$checksum_type" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
       log_rest 2 "error getting checksum: $checksum"
       return 1
     fi
-    ;;
-  "x-amz-checksum-crc64nvme")
-    if ! checksum=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc64nvme" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
-      log 2 "error calculating checksum: $checksum"
-      return 1
-    fi
-    ;;
-  "x-amz-checksum-sha256")
-    checksum="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
-    ;;
-  "x-amz-checksum-sha1")
-    checksum="$(sha1sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
-    ;;
-  "x-amz-checksum-crc32")
-    checksum="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
-    ;;
-  *)
-    log_rest 2 "invalid trailer type: '$TRAILER'"
-    return 1
-    ;;
-  esac
-  signature_string="$TRAILER:$checksum"
+  else
+    checksum="$CHECKSUM"
+  fi
+  signature_string="$trailer:$checksum"
   trailer_payload_hash="$(echo "$signature_string" | sha256sum | awk '{print $1}')"
+  return 0
 }
 
 get_chunk_sizes() {
@@ -317,7 +303,7 @@ build_chunk() {
 
 build_trailer() {
   log_rest 5 "payload hash: $payload_hash"
-  final_sts_data="$initial_trailer_sts_data
+  final_sts_data="$trailer_sts_data
 $signature
 $trailer_payload_hash"
   log_rest 5 "$final_sts_data"
@@ -422,7 +408,7 @@ load_parameters
 
 if ! calculate_checksum; then
   log_rest 2 "error calculating trailer checksum"
-  return 1
+  exit 1
 fi
 if ! get_file_size_and_content_length; then
   log_rest 2 "error getting file size and content length"

tests/rest_scripts/put_object_retention.sh

@@ -26,12 +26,16 @@ key="$OBJECT_KEY"
 mode="$RETENTION_MODE"
 # shellcheck disable=SC2153
 retain_until_date="$RETAIN_UNTIL_DATE"
+# shellcheck disable=SC2153
+omit_payload="${OMIT_PAYLOAD:=false}"
 
-payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
-<Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
-  <Mode>$mode</Mode>
-  <RetainUntilDate>$retain_until_date</RetainUntilDate>
-</Retention>"
+if [ "$omit_payload" == "false" ]; then
+  payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
+  <Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
+    <Mode>$mode</Mode>
+    <RetainUntilDate>$retain_until_date</RetainUntilDate>
+  </Retention>"
+fi
 
 payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
 content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64)
@@ -55,8 +59,10 @@ curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_
 -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
 -H "\"Content-MD5: $content_md5\""
 -H "\"x-amz-content-sha256: $payload_hash\""
--H "\"x-amz-date: $current_date_time\""
--d "\"${payload//\"/\\\"}\""
--o "$OUTPUT_FILE")
+-H "\"x-amz-date: $current_date_time\"")
+if [ "$omit_payload" == "false" ]; then
+  curl_command+=(-d "\"${payload//\"/\\\"}\"")
+fi
+curl_command+=(-o "$OUTPUT_FILE")
 # shellcheck disable=SC2154
 eval "${curl_command[*]}" 2>&1