test: REST HeadObject

tests/util/util_acl.sh

@@ -388,3 +388,60 @@ put_canned_acl_rest() {
   fi
   return 0
 }
+
+# param: bucket name
+# return 0 for success, 1 for failure
+check_ownership_rule_and_reset_acl() {
+  if [ $# -ne 1 ]; then
+    log 2 "'check_ownership_rule_and_reset_acl' requires bucket name"
+    return 1
+  fi
+  if ! get_bucket_ownership_controls "$1"; then
+    log 2 "error getting bucket ownership controls"
+    return 1
+  fi
+  # shellcheck disable=SC2154
+  if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then
+    log 2 "error getting object ownership rule: $object_ownership_rule"
+    return 1
+  fi
+  if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then
+    log 2 "error resetting bucket ACL"
+    return 1
+  fi
+}
+
+# param: bucket name
+# return 1 for failure, 0 for success
+get_object_ownership_rule_and_update_acl() {
+  if [ $# -ne 1 ]; then
+    log 2 "'get_object_ownership_rule_and_update_acl' requires bucket name"
+    return 1
+  fi
+  if ! get_object_ownership_rule "$1"; then
+    log 2 "error getting object ownership rule"
+    return 1
+  fi
+  log 5 "object ownership rule: $object_ownership_rule"
+  if [[ "$object_ownership_rule" != "BucketOwnerEnforced" ]] && ! put_bucket_canned_acl "$1" "private"; then
+    log 2 "error resetting bucket ACLs"
+    return 1
+  fi
+}
+
+# get object acl
+# param:  object path
+# export acl for success, return 1 for error
+get_object_acl() {
+  if [ $# -ne 2 ]; then
+    log 2 "object ACL command missing object name"
+    return 1
+  fi
+  local exit_code=0
+  acl=$(aws --no-verify-ssl s3api get-object-acl --bucket "$1" --key "$2" 2>&1) || exit_code="$?"
+  if [ $exit_code -ne 0 ]; then
+    log 2 "Error getting object ACLs: $acl"
+    return 1
+  fi
+  export acl
+}

tests/util/util_attributes.sh

@@ -112,4 +112,21 @@ add_and_check_checksum() {
     log 2 "empty checksum"
     return 1
   fi
+}
+
+get_etag_attribute_rest() {
+  if [ $# -ne 3 ]; then
+    log 2 "'get_etag_attribute_rest' requires bucket name, object key, expected etag"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" ATTRIBUTES="ETag" OUTPUT_FILE="$TEST_FILE_FOLDER/attributes.txt" ./tests/rest_scripts/get_object_attributes.sh); then
+    log 2 "error attempting to get object info: $result"
+    return 1
+  fi
+  log 5 "attributes: $(cat "$TEST_FILE_FOLDER/attributes.txt")"
+  if ! check_xml_element "$TEST_FILE_FOLDER/attributes.txt" "$3" "GetObjectAttributesResponse" "ETag"; then
+    log 2 "etag mismatch"
+    return 1
+  fi
+  return 0
 }

tests/util/util_aws.sh

@@ -51,4 +51,18 @@ abort_all_multipart_uploads() {
     fi
   done <<< "$lines"
   return 0
-}
+}
+
+remove_insecure_request_warning() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "remove insecure request warning requires input lines"
+    return 1
+  fi
+  parsed_output=()
+  while IFS= read -r line; do
+    if [[ $line != *InsecureRequestWarning* ]]; then
+      parsed_output+=("$line")
+    fi
+  done <<< "$1"
+  export parsed_output
+}

tests/util/util_bucket.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 
+source ./tests/util/util_acl.sh
 source ./tests/util/util_multipart_abort.sh
+source ./tests/util/util_policy.sh
+source ./tests/util/util_retention.sh
 
 # recursively delete an AWS bucket
 # param:  client, bucket name

tests/util/util_head_object.sh

@@ -69,4 +69,22 @@ get_and_verify_metadata() {
     return 1
   fi
   return 0
+}
+
+get_etag_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'get_etag_rest' requires bucket name, object key"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/head_object.txt" ./tests/rest_scripts/head_object.sh); then
+    log 2 "error attempting to get object info: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "response code '$result', data: $(cat "$TEST_FILE_FOLDER/head_object.txt")"
+    return 1
+  fi
+  log 5 "head object data: $(cat "$TEST_FILE_FOLDER/head_object.txt")"
+  etag_value=$(grep "E[Tt]ag:" "$TEST_FILE_FOLDER/head_object.txt" | sed -n 's/E[Tt]ag: "\([^"]*\)"/\1/p' | tr -d '\r')
+  echo "$etag_value"
 }

tests/util/util_multipart.sh

@@ -295,3 +295,43 @@ create_upload_finish_wrong_etag() {
   fi
   return 0
 }
+
+setup_multipart_upload_with_params() {
+  if [ $# -ne 2 ]; then
+    log 2 "'setup_multipart_upload_with_params' requires bucket name, file name"
+    return 1
+  fi
+  os_name="$(uname)"
+  if [[ "$os_name" == "Darwin" ]]; then
+    now=$(date -u +"%Y-%m-%dT%H:%M:%S")
+    later=$(date -j -v +15S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S")
+  else
+    now=$(date +"%Y-%m-%dT%H:%M:%S")
+    later=$(date -d "$now 15 seconds" +"%Y-%m-%dT%H:%M:%S")
+  fi
+
+  if ! create_test_files "$2"; then
+    log 2 "error creating test file"
+    return 1
+  fi
+
+  if ! result=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER/$2" bs=5M count=1 2>&1); then
+    log 2 "error creating large file: $result"
+    return 1
+  fi
+
+  if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
+    log 2 "error cleaning up bucket"
+    return 1
+  fi
+  # in static bucket config, bucket will still exist
+  if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
+    if ! create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"; then
+      log 2 "error creating bucket with object lock enabled"
+      return 1
+    fi
+  fi
+  log 5 "later in function: $later"
+  echo "$later"
+  return 0
+}

tests/util/util_object.sh

@@ -47,46 +47,6 @@ source ./tests/commands/upload_part_copy.sh
 source ./tests/commands/upload_part.sh
 source ./tests/util/util_users.sh
 
-# params:  bucket name
-# return 0 for success, 1 for error
-add_governance_bypass_policy() {
-  if [[ $# -ne 1 ]]; then
-    log 2 "'add governance bypass policy' command requires bucket name"
-    return 1
-  fi
-  cat <<EOF > "$TEST_FILE_FOLDER/policy-bypass-governance.txt"
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-       "Effect": "Allow",
-       "Principal": "*",
-       "Action": "s3:BypassGovernanceRetention",
-       "Resource": "arn:aws:s3:::$1/*"
-    }
-  ]
-}
-EOF
-  if ! put_bucket_policy "s3api" "$1" "$TEST_FILE_FOLDER/policy-bypass-governance.txt"; then
-    log 2 "error putting governance bypass policy"
-    return 1
-  fi
-  return 0
-}
-
-log_bucket_policy() {
-  if [ $# -ne 1 ]; then
-    log 2 "'log_bucket_policy' requires bucket name"
-    return
-  fi
-  if ! get_bucket_policy "s3api" "$1"; then
-    log 2 "error getting bucket policy"
-    return
-  fi
-  # shellcheck disable=SC2154
-  log 5 "BUCKET POLICY: $bucket_policy"
-}
-
 # param: bucket name
 # return 0 for success, 1 for failure
 list_and_delete_objects() {
@@ -114,28 +74,6 @@ list_and_delete_objects() {
   return 0
 }
 
-# param: bucket name
-# return 0 for success, 1 for failure
-check_ownership_rule_and_reset_acl() {
-  if [ $# -ne 1 ]; then
-    log 2 "'check_ownership_rule_and_reset_acl' requires bucket name"
-    return 1
-  fi
-  if ! get_bucket_ownership_controls "$1"; then
-    log 2 "error getting bucket ownership controls"
-    return 1
-  fi
-  # shellcheck disable=SC2154
-  if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then
-    log 2 "error getting object ownership rule: $object_ownership_rule"
-    return 1
-  fi
-  if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then
-    log 2 "error resetting bucket ACL"
-    return 1
-  fi
-}
-
 check_object_lock_config() {
   if [ $# -ne 1 ]; then
     log 2 "'check_object_lock_config' requires bucket name"
@@ -174,76 +112,6 @@ clear_object_in_bucket() {
   return 0
 }
 
-# params: bucket, object, possible WORM error after deletion attempt
-# return 0 for success, 1 for no WORM protection, 2 for error
-check_for_and_remove_worm_protection() {
-  if [ $# -ne 3 ]; then
-    log 2 "'check_for_and_remove_worm_protection' command requires bucket, object, error"
-    return 2
-  fi
-
-  if [[ $3 == *"WORM"* ]]; then
-    log 5 "WORM protection found"
-    if ! put_object_legal_hold "$1" "$2" "OFF"; then
-      log 2 "error removing object legal hold"
-      return 2
-    fi
-    sleep 1
-    if [[ $LOG_LEVEL_INT -ge 5 ]]; then
-      log_worm_protection "$1" "$2"
-    fi
-    if ! add_governance_bypass_policy "$1"; then
-      log 2 "error adding new governance bypass policy"
-      return 2
-    fi
-    if ! delete_object_bypass_retention "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then
-      log 2 "error deleting object after legal hold removal"
-      return 2
-    fi
-  else
-    log 5 "no WORM protection found"
-    return 1
-  fi
-  return 0
-}
-
-# params: bucket name, object
-log_worm_protection() {
-  if ! get_object_legal_hold "$1" "$2"; then
-    log 2 "error getting object legal hold status"
-    return
-  fi
-  # shellcheck disable=SC2154
-  log 5 "LEGAL HOLD: $legal_hold"
-  if ! get_object_retention "$1" "$2"; then
-    log 2 "error getting object retention"
-    # shellcheck disable=SC2154
-    if [[ $get_object_retention_error != *"NoSuchObjectLockConfiguration"* ]]; then
-      return
-    fi
-  fi
-  # shellcheck disable=SC2154
-  log 5 "RETENTION: $retention"
-}
-
-# param: bucket name
-# return 1 for failure, 0 for success
-get_object_ownership_rule_and_update_acl() {
-  if [ $# -ne 1 ]; then
-    log 2 "'get_object_ownership_rule_and_update_acl' requires bucket name"
-    return 1
-  fi
-  if ! get_object_ownership_rule "$1"; then
-    log 2 "error getting object ownership rule"
-    return 1
-  fi
-  log 5 "object ownership rule: $object_ownership_rule"
-  if [[ "$object_ownership_rule" != "BucketOwnerEnforced" ]] && ! put_bucket_canned_acl "$1" "private"; then
-    log 2 "error resetting bucket ACLs"
-    return 1
-  fi
-}
-
 # check if object exists on S3 via gateway
 # param:  command, object path
 # return 0 for true, 1 for false, 2 for error
@@ -359,20 +227,6 @@ check_and_put_object() {
   return 0
 }
 
-remove_insecure_request_warning() {
-  if [[ $# -ne 1 ]]; then
-    log 2 "remove insecure request warning requires input lines"
-    return 1
-  fi
-  parsed_output=()
-  while IFS= read -r line; do
-    if [[ $line != *InsecureRequestWarning* ]]; then
-      parsed_output+=("$line")
-    fi
-  done <<< "$1"
-  export parsed_output
-}
-
 # check if object info (etag) is accessible
 # param:  path of object
 # return 0 for yes, 1 for no, 2 for error
@@ -394,23 +248,6 @@ object_is_accessible() {
   return 0
 }
 
-# get object acl
-# param:  object path
-# export acl for success, return 1 for error
-get_object_acl() {
-  if [ $# -ne 2 ]; then
-    log 2 "object ACL command missing object name"
-    return 1
-  fi
-  local exit_code=0
-  acl=$(aws --no-verify-ssl s3api get-object-acl --bucket "$1" --key "$2" 2>&1) || exit_code="$?"
-  if [ $exit_code -ne 0 ]; then
-    log 2 "Error getting object ACLs: $acl"
-    return 1
-  fi
-  export acl
-}
-
 # copy a file to/from S3
 # params:  source, destination
 # return 0 for success, 1 for failure

tests/util/util_policy.sh

@@ -330,3 +330,16 @@ put_and_check_policy_rest() {
   fi
   return 0
 }
+
+log_bucket_policy() {
+  if [ $# -ne 1 ]; then
+    log 2 "'log_bucket_policy' requires bucket name"
+    return
+  fi
+  if ! get_bucket_policy "s3api" "$1"; then
+    log 2 "error getting bucket policy"
+    return
+  fi
+  # shellcheck disable=SC2154
+  log 5 "BUCKET POLICY: $bucket_policy"
+}

tests/util/util_retention.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# params:  bucket name
+# return 0 for success, 1 for error
+add_governance_bypass_policy() {
+  if [[ $# -ne 1 ]]; then
+    log 2 "'add governance bypass policy' command requires bucket name"
+    return 1
+  fi
+  cat <<EOF > "$TEST_FILE_FOLDER/policy-bypass-governance.txt"
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+       "Effect": "Allow",
+       "Principal": "*",
+       "Action": "s3:BypassGovernanceRetention",
+       "Resource": "arn:aws:s3:::$1/*"
+    }
+  ]
+}
+EOF
+  if ! put_bucket_policy "s3api" "$1" "$TEST_FILE_FOLDER/policy-bypass-governance.txt"; then
+    log 2 "error putting governance bypass policy"
+    return 1
+  fi
+  return 0
+}
+
+# params: bucket, object, possible WORM error after deletion attempt
+# return 0 for success, 1 for no WORM protection, 2 for error
+check_for_and_remove_worm_protection() {
+  if [ $# -ne 3 ]; then
+    log 2 "'check_for_and_remove_worm_protection' command requires bucket, object, error"
+    return 2
+  fi
+
+  if [[ $3 == *"WORM"* ]]; then
+    log 5 "WORM protection found"
+    if ! put_object_legal_hold "$1" "$2" "OFF"; then
+      log 2 "error removing object legal hold"
+      return 2
+    fi
+    sleep 1
+    if [[ $LOG_LEVEL_INT -ge 5 ]]; then
+      log_worm_protection "$1" "$2"
+    fi
+    if ! add_governance_bypass_policy "$1"; then
+      log 2 "error adding new governance bypass policy"
+      return 2
+    fi
+    if ! delete_object_bypass_retention "$1" "$2" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"; then
+      log 2 "error deleting object after legal hold removal"
+      return 2
+    fi
+  else
+    log 5 "no WORM protection found"
+    return 1
+  fi
+  return 0
+}
+
+# params: bucket name, object
+log_worm_protection() {
+  if ! get_object_legal_hold "$1" "$2"; then
+    log 2 "error getting object legal hold status"
+    return
+  fi
+  # shellcheck disable=SC2154
+  log 5 "LEGAL HOLD: $legal_hold"
+  if ! get_object_retention "$1" "$2"; then
+    log 2 "error getting object retention"
+    # shellcheck disable=SC2154
+    if [[ $get_object_retention_error != *"NoSuchObjectLockConfiguration"* ]]; then
+      return
+    fi
+  fi
+  # shellcheck disable=SC2154
+  log 5 "RETENTION: $retention"
+}