test: initial checksum tests, REST script additions

tests/commands/get_object_lock_configuration.sh

@@ -36,35 +36,12 @@ get_object_lock_configuration_rest() {
     log 2 "'get_object_lock_configuration_rest' requires bucket name"
     return 1
   fi
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-object-lock=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object-lock-config.txt" ./tests/rest_scripts/get_object_lock_config.sh); then
+    log 2 "error getting lock configuration: $result"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1)
-  log 5 "reply: $reply"
-  if [[ "$reply" != "200" ]]; then
-    log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
+  if [[ "$result" != "200" ]]; then
+    log 2 "expected '200', returned '$result': $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
     return 1
   fi
   return 0

tests/commands/list_objects.sh

@@ -138,35 +138,19 @@ list_objects_rest() {
     log 2 "'list_objects_rest' requires bucket name"
     return 1
   fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  log 5 "canonical request: $canonical_request"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  log 5 "bucket name: $1"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error listing objects: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/objects.txt"))"
+    return 1
+  fi
+  # shellcheck disable=SC2034
+  reply=$(cat "$TEST_FILE_FOLDER/objects.txt")
+  if ! parse_objects_list_rest; then
+    log 2 "error parsing list objects"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks "$header://$aws_endpoint_url_address/$1" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" 2>&1)
-  log 5 "reply: $reply"
-  parse_objects_list_rest
 }

tests/rest_scripts/get_object_lock_config.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Fields
+
+source ./tests/rest_scripts/rest.sh
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+canonical_request_data=("GET" "/$bucket_name" "object-lock=" "host:$host")
+canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
+if ! build_canonical_request "${canonical_request_data[@]}"; then
+  log_rest 2 "error building request"
+  exit 1
+fi
+echo "$canonical_request" > "cr.txt"
+
+# shellcheck disable=SC2119
+create_canonical_hash_sts_and_signature
+
+# shellcheck disable=SC2154
+curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?object-lock")
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$year_month_day/$AWS_REGION/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
+curl_command+=(-o "$OUTPUT_FILE")
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/head_object.sh

@@ -22,21 +22,18 @@ source ./tests/rest_scripts/rest.sh
 bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2154
 key="$OBJECT_KEY"
+# shellcheck disable=SC2153
+version_id="$VERSION_ID"
 
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
 
-#x-amz-object-attributes:ETag
-canonical_request="HEAD
-/$bucket_name/$key
-
-host:$host
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-canonical_request_data=("HEAD" "/$bucket_name/$key" "" "host:$host")
+canonical_request_data=("HEAD" "/$bucket_name/$key")
+if [ "$version_id" != "" ]; then
+  canonical_request_data+=("versionId=$version_id")
+else
+  canonical_request_data+=("")
+fi
+canonical_request_data+=("host:$host")
 if [ "$CHECKSUM" == "true" ]; then
   canonical_request_data+=("x-amz-checksum-mode:ENABLED")
 fi
@@ -48,7 +45,11 @@ fi
 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
-curl_command+=(curl -ksI -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key"
+url="$AWS_ENDPOINT_URL/$bucket_name/$key"
+if [ "$version_id" != "" ]; then
+  url+="?versionId=$version_id"
+fi
+curl_command+=(curl -ksI -w "\"%{http_code}\"" "$url"
 -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
 curl_command+=("${header_fields[@]}")
 curl_command+=(-o "$OUTPUT_FILE")

tests/rest_scripts/put_object.sh

@@ -25,15 +25,32 @@ bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2153
 key="$OBJECT_KEY"
 # shellcheck disable=SC2153,SC2154
-checksum="$CHECKSUM"
+checksum_type="$CHECKSUM_TYPE"
+# shellcheck disable=SC2153
+payload="$PAYLOAD"
+
+# use this parameter to check incorrect checksums
+# shellcheck disable=SC2153,SC2154
+checksum_hash="$CHECKSUM"
 
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-payload_hash="$(sha256sum "$data_file" | awk '{print $1}')"
+if [ "$payload" == "" ]; then
+  payload_hash="$(sha256sum "$data_file" | awk '{print $1}')"
+else
+  payload_hash="$payload"
+fi
 
 cr_data=("PUT" "/$bucket_name/$key" "" "host:$host")
-if [ "$checksum" == "true" ]; then
-  checksum_hash="$(echo -n "$payload_hash" | xxd -r -p | base64)"
+if [ "$checksum_type" == "sha256" ]; then
+  if [ -z "$checksum_hash" ]; then
+    checksum_hash="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
+  fi
   cr_data+=("x-amz-checksum-sha256:$checksum_hash")
+elif [ "$checksum_type" == "crc32" ]; then
+  if [ -z "$checksum_hash" ]; then
+    checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
+  fi
+  cr_data+=("x-amz-checksum-crc32:$checksum_hash")
 fi
 cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
 build_canonical_request "${cr_data[@]}"

tests/run.sh

@@ -149,6 +149,8 @@ run_suite() {
         exit_code=1
       elif ! "$HOME"/bin/bats ./tests/test_rest_checksum.sh; then
         exit_code=1
+      elif ! "$HOME"/bin/bats ./tests/test_rest_versioning.sh; then
+        exit_code=1
       fi
       ;;
     s3api-user)

tests/test_rest.sh

@@ -147,28 +147,6 @@ export RUN_USERS=true
   assert_success
 }
 
-@test "REST - check, enable, suspend versioning" {
-  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
-  assert_success
-
-  log 5 "get versioning"
-
-  run check_versioning_status_rest "$BUCKET_ONE_NAME" ""
-  assert_success
-
-  run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run check_versioning_status_rest "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Suspended"
-  assert_success
-
-  run check_versioning_status_rest "$BUCKET_ONE_NAME" "Suspended"
-  assert_success
-}
-
 @test "test_rest_set_get_lock_config" {
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
@@ -189,66 +167,6 @@ export RUN_USERS=true
   assert_success
 }
 
-@test "test_rest_versioning" {
-  test_file="test_file"
-  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true"
-  assert_success
-
-  run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true"
-  assert_success
-
-  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "2" "true" "false" "false" "true"
-  assert_success
-}
-
-@test "versioning - add version, then delete and check for marker" {
-  test_file="test_file"
-  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run delete_object_rest "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run check_versions_after_file_deletion "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-}
-
-@test "versioning - retrieve after delete" {
-  test_file="test_file"
-  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
-  assert_failure
-}
-
 @test "REST - legal hold, get without config" {
   test_file="test_file"
   run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
@@ -534,3 +452,72 @@ export RUN_USERS=true
   run verify_object_not_found "$BUCKET_ONE_NAME" "$test_file_two"
   assert_success
 }
+
+@test "REST - put object w/STREAMING-AWS4-HMAC-SHA256-PAYLOAD without content length" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1043"
+  fi
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object_rest_chunked_payload_type_without_content_length "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - HeadObject does not return 405 with versioning, after file deleted" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1029"
+  fi
+  if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
+    skip
+  fi
+  run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  # in static bucket config, bucket will still exist
+  if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
+    run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
+    assert_success
+  fi
+
+  test_file="test_file"
+  run create_test_files "$test_file"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run verify_object_not_found "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - HeadObject returns 405 when querying DeleteMarker" {
+  if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
+    skip
+  fi
+  run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  # in static bucket config, bucket will still exist
+  if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
+    run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
+    assert_success
+  fi
+
+  test_file="test_file"
+  run create_test_files "$test_file"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run get_delete_marker_and_verify_405 "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}

tests/test_rest_checksum.sh

@@ -22,16 +22,52 @@ source ./tests/util/util_head_object.sh
 source ./tests/util/util_setup.sh
 
 @test "REST - HeadObject returns x-amz-checksum-sha256" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/1070"
-  fi
   test_file="test_file"
   run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
   assert_success
 
-  run put_object_rest_sha256_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256"
   assert_success
 
-  run check_checksum_rest "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
+  run check_checksum_rest_sha256 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
   assert_success
-}
+}
+
+@test "REST - PutObject rejects invalid sha256 checksum" {
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object_rest_sha256_invalid "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - PutObject rejects incorrect sha256 checksum" {
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object_rest_sha256_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - crc32 checksum - success" {
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc32"
+  assert_success
+}
+
+@test "REST - crc32 checksum - correct" {
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc32"
+  assert_success
+
+  run check_checksum_rest_crc32 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
+  assert_success
+}

tests/test_rest_versioning.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+load ./bats-support/load
+load ./bats-assert/load
+
+source ./tests/setup.sh
+source ./tests/commands/get_object.sh
+source ./tests/commands/put_object.sh
+source ./tests/util/util_rest.sh
+source ./tests/util/util_setup.sh
+
+@test "REST - check, enable, suspend versioning" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  log 5 "get versioning"
+
+  run check_versioning_status_rest "$BUCKET_ONE_NAME" ""
+  assert_success
+
+  run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Enabled"
+  assert_success
+
+  run check_versioning_status_rest "$BUCKET_ONE_NAME" "Enabled"
+  assert_success
+
+  run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Suspended"
+  assert_success
+
+  run check_versioning_status_rest "$BUCKET_ONE_NAME" "Suspended"
+  assert_success
+}
+
+@test "test_rest_versioning" {
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true"
+  assert_success
+
+  run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
+  assert_success
+
+  run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true"
+  assert_success
+
+  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "2" "true" "false" "false" "true"
+  assert_success
+}
+
+@test "versioning - add version, then delete and check for marker" {
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
+  assert_success
+
+  run delete_object_rest "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run check_versions_after_file_deletion "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "versioning - retrieve after delete" {
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
+  assert_success
+
+  run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
+  assert_failure
+}

tests/util/util_head_object.sh

@@ -122,8 +122,8 @@ verify_object_exists() {
 }
 
 check_checksum_rest() {
-  if [ $# -ne 3 ]; then
-    log 2 "'check_checksum_rest' requires bucket, file, local file"
+  if [ $# -ne 4 ]; then
+    log 2 "'check_checksum_rest' requires bucket, file, expected checksum, header key"
     return 1
   fi
   if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" CHECKSUM="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/head_object.sh 2>&1); then
@@ -134,10 +134,35 @@ check_checksum_rest() {
     log 2 "expected response code '200', was '$result'"
     return 1
   fi
-  head_checksum=$(grep "x-amz-checksum-sha256" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//')
-  file_checksum="$(sha256sum "$3" | awk '{print $1}' | xxd -r -p | base64)"
-  if [ "$file_checksum" != "$head_checksum" ]; then
+  head_checksum=$(grep -i "$4" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//')
+  if [ "$3" != "$head_checksum" ]; then
     log 2 "'checksum mismatch (head '$head_checksum', local '$file_checksum')"
     return 1
   fi
 }
+
+check_checksum_rest_sha256() {
+  if [ $# -ne 3 ]; then
+    log 2 "'check_checksum_rest_sha256' requires bucket, file, local file"
+    return 1
+  fi
+  file_checksum="$(sha256sum "$3" | awk '{print $1}' | xxd -r -p | base64)"
+  if ! check_checksum_rest "$1" "$2" "$file_checksum" "x-amz-checksum-sha256"; then
+    log 2 "error checking checksum"
+    return 1
+  fi
+  return 0
+}
+
+check_checksum_rest_crc32() {
+  if [ $# -ne 3 ]; then
+    log 2 "'check_checksum_rest_crc32' requires bucket, file, local file"
+    return 1
+  fi
+  file_checksum="$(gzip -c -1 "$3" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
+  if ! check_checksum_rest "$1" "$2" "$file_checksum" "x-amz-checksum-crc32"; then
+    log 2 "error checking checksum"
+    return 1
+  fi
+  return 0
+}

tests/util/util_list_objects.sh

@@ -328,3 +328,31 @@ list_objects_v1_check_nextmarker_empty() {
   fi
   return 0
 }
+
+get_delete_marker_and_verify_405() {
+  if [ $# -ne 2 ]; then
+    log 2 "'get_delete_marker_and_verify_405' requires bucket name, file name"
+    return 1
+  fi
+  if ! list_object_versions_rest "$1"; then
+    log 2 "error listing REST object versions"
+    return 1
+  fi
+  log 5 "versions: $(cat "$TEST_FILE_FOLDER/object_versions.txt")"
+
+  if ! version_id=$(xmllint --xpath "//*[local-name()=\"DeleteMarker\"]/*[local-name()=\"VersionId\"]/text()" "$TEST_FILE_FOLDER/object_versions.txt" 2>&1); then
+    log 2 "error getting XML value: $version_id"
+    return 1
+  fi
+  log 5 "xml val: $version_id"
+
+  if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then
+    log 2 "error getting result: $result"
+    return 1
+  fi
+  if [ "$result" != "405" ]; then
+    log 2 "expected '405', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+    return 1
+  fi
+  return 0
+}

tests/util/util_lock_config.sh

@@ -96,7 +96,7 @@ check_no_object_lock_config_rest() {
   fi
   log 5 "object lock config: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
   # shellcheck disable=SC2154
-  if [[ "$reply" != "404" ]]; then
+  if [[ "$result" != "404" ]]; then
     log 2 "incorrect response code: $reply"
     return 1
   fi

tests/util/util_object.sh

@@ -297,12 +297,32 @@ list_and_check_directory_obj() {
   return 0
 }
 
-put_object_rest_sha256_checksum() {
-  if [ $# -ne 3 ]; then
-    log 2 "'put_object_rest_sha256_checksum' requires data file, bucket name, key"
+check_sha256_invalid_or_incorrect() {
+  if [ $# -ne 5 ]; then
+    log 2 "'check_sha256_invalid_or_incorrect' requires data file, bucket name, key, checksum, expected error"
     return 1
   fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM="true" ./tests/rest_scripts/put_object.sh 2>&1); then
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="sha256" CHECKSUM="$4" ./tests/rest_scripts/put_object.sh 2>&1); then
+    log 2 "error: $result"
+    return 1
+  fi
+  if [ "$result" != "400" ]; then
+    log 2 "expected response code of '400', was '$result' (response: $(cat "$TEST_FILE_FOLDER/result.txt")"
+    return 1
+  fi
+  if ! check_xml_element "$TEST_FILE_FOLDER/result.txt" "$5" "Error" "Message"; then
+    log 2 "xml error message mismatch"
+    return 1
+  fi
+  return 0
+}
+
+put_object_rest_checksum() {
+  if [ $# -ne 4 ]; then
+    log 2 "'put_object_rest_sha256_checksum' requires data file, bucket name, key, checksum type"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="$4" ./tests/rest_scripts/put_object.sh 2>&1); then
     log 2 "error: $result"
     return 1
   fi
@@ -313,3 +333,49 @@ put_object_rest_sha256_checksum() {
   log 5 "result: $(cat "$TEST_FILE_FOLDER/result.txt")"
   return 0
 }
+
+put_object_rest_sha256_invalid() {
+  if [ $# -ne 3 ]; then
+    log 2 "'put_object_rest_sha256_invalid' requires data file, bucket name, key"
+    return 1
+  fi
+  if ! check_sha256_invalid_or_incorrect "$1" "$2" "$3" "dummy" "Value for x-amz-checksum-sha256 header is invalid."; then
+    log 2 "error checking checksum"
+    return 1
+  fi
+  return 0
+}
+
+put_object_rest_sha256_incorrect() {
+  if [ $# -ne 3 ]; then
+    log 2 "'put_object_rest_sha256_invalid' requires data file, bucket name, key"
+    return 1
+  fi
+  if [ "$DIRECT" == "true" ]; then
+    error_message="The SHA256 you specified did not match the calculated checksum."
+  else
+    error_message="The sha256 you specified did not match the calculated checksum."
+  fi
+  incorrect_checksum="$(echo -n "dummy" | sha256sum | awk '{print $1}' | xxd -r -p | base64)"
+  if ! check_sha256_invalid_or_incorrect "$1" "$2" "$3" "$incorrect_checksum" "$error_message"; then
+    log 2 "error checking checksum"
+    return 1
+  fi
+  return 0
+}
+
+put_object_rest_chunked_payload_type_without_content_length() {
+  if [ $# -ne 3 ]; then
+    log 2 "'put_object_rest_diff_payload_type' requires data file, bucket name, key"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" PAYLOAD="STREAMING-AWS4-HMAC-SHA256-PAYLOAD" ./tests/rest_scripts/put_object.sh 2>&1); then
+    log 2 "error: $result"
+    return 1
+  fi
+  if [ "$result" != "411" ]; then
+    log 2 "expected response code of '411', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
+    return 1
+  fi
+  return 0
+}

tests/util/util_xml.sh

@@ -19,4 +19,4 @@ check_xml_element() {
     return 1
   fi
   return 0
-}
+}