test: GetBucketAcl, cleanup

tests/rest_scripts/get_bucket_acl.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="GET
+/$bucket_name
+acl=
+host:$host
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?acl="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
+-H "\"x-amz-date: $current_date_time\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/test_rest.sh

@@ -29,6 +29,7 @@ source ./tests/commands/put_object_tagging.sh
 source ./tests/logger.sh
 source ./tests/setup.sh
 source ./tests/util/util.sh
+source ./tests/util/util_acl.sh
 source ./tests/util/util_attributes.sh
 source ./tests/util/util_legal_hold.sh
 source ./tests/util/util_list_buckets.sh
@@ -189,7 +190,6 @@ export RUN_USERS=true
 }
 
 @test "test_rest_versioning" {
-  skip "https://github.com/versity/versitygw/issues/864"
   test_file="test_file"
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
@@ -222,7 +222,6 @@ export RUN_USERS=true
 }
 
 @test "versioning - add version, then delete and check for marker" {
-  skip "https://github.com/versity/versitygw/issues/864"
   test_file="test_file"
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
@@ -442,4 +441,15 @@ export RUN_USERS=true
 
   run put_and_check_policy_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/policy_file.txt" "Allow" "$USERNAME_ONE" "s3:PutBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME"
   assert_success
+}
+
+@test "REST - get ACL" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/971"
+  fi
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run get_and_check_acl_rest "$BUCKET_ONE_NAME"
+  assert_success
 }

tests/test_s3api_multipart.sh

@@ -18,6 +18,7 @@ source ./tests/setup.sh
 source ./tests/test_s3api_root_inner.sh
 source ./tests/util/util_file.sh
 source ./tests/util/util_multipart.sh
+source ./tests/util/util_multipart_abort.sh
 source ./tests/util/util_tags.sh
 source ./tests/commands/get_object.sh
 source ./tests/commands/put_object.sh

tests/test_s3api_policy.sh

@@ -20,6 +20,7 @@ source ./tests/test_s3api_policy_bucket.sh
 source ./tests/test_s3api_policy_multipart.sh
 source ./tests/test_s3api_policy_object.sh
 source ./tests/util/util_multipart.sh
+source ./tests/util/util_multipart_abort.sh
 source ./tests/util/util_file.sh
 source ./tests/util/util_policy.sh
 source ./tests/util/util_tags.sh

tests/test_s3api_policy_multipart.sh

@@ -14,6 +14,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+source ./tests/util/util_multipart_abort.sh
+
 test_s3api_policy_abort_multipart_upload() {
   policy_file="policy_file"
   test_file="test_file"

tests/util/util_acl.sh

@@ -199,3 +199,47 @@ get_check_acl_after_policy() {
     fi
   fi
 }
+
+get_and_check_acl_rest() {
+  if [ $# -ne 1 ]; then
+    log 2 "'get_and_check_acl_rest' requires bucket name"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/acl.txt" ./tests/rest_scripts/get_bucket_acl.sh); then
+    log 2 "error attempting to get bucket ACL response: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "get acl returned code '$result' (message: $(cat "$TEST_FILE_FOLDER/acl.txt"))"
+    return 1
+  fi
+  log 5 "acl: $(cat "$TEST_FILE_FOLDER/acl.txt")"
+  if ! access_control_policy=$(xmllint --xpath '//*[local-name()="AccessControlPolicy"]' "$TEST_FILE_FOLDER/acl.txt" 2>&1); then
+    log 2 "error getting access control policy: $access_control_policy"
+    return 1
+  fi
+  if ! owner=$(echo "$access_control_policy" | xmllint --xpath '//*[local-name()="Owner"]' - 2>&1); then
+    log 2 "error getting owner information: $owner"
+    return 1
+  fi
+  if [ "$DIRECT" == "true" ]; then
+    if ! display_name=$(echo "$owner" | xmllint --xpath '//*[local-name()="DisplayName"]/text()' - 2>&1); then
+      log 2 "error getting display name: $display_name"
+      return 1
+    fi
+    if [ "$display_name" != "$DIRECT_DISPLAY_NAME" ]; then
+      log 2 "display name mismatch (expected '$DIRECT_DISPLAY_NAME', actual '$display_name')"
+      return 1
+    fi
+  else
+    if ! id=$(echo "$owner" | xmllint --xpath '//*[local-name()="ID"]/text()' - 2>&1); then
+      log 2 "error getting ID: $id"
+      return 1
+    fi
+    if [ "$id" != "$AWS_ACCESS_KEY_ID" ]; then
+      log 2 "ID mismatch"
+      return 1
+    fi
+  fi
+  return 0
+}

tests/util/util_bucket.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+source ./tests/util/util_multipart_abort.sh
+
 # recursively delete an AWS bucket
 # param:  client, bucket name
 # fail if error

tests/util/util_multipart.sh

@@ -476,68 +476,6 @@ multipart_upload_with_params() {
   return 0
 }
 
-# run upload, then abort it
-# params:  bucket, key, local file location, number of parts to split into before uploading
-# return 0 for success, 1 for failure
-run_then_abort_multipart_upload() {
-  if [ $# -ne 4 ]; then
-    log 2 "run then abort multipart upload command missing bucket, key, file, and/or part count"
-    return 1
-  fi
-
-  if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then
-    log 2 "error performing pre-completion multipart upload"
-    return 1
-  fi
-
-  if ! abort_multipart_upload "$1" "$2" "$upload_id"; then
-    log 2 "error aborting multipart upload"
-    return 1
-  fi
-  return 0
-}
-
-# param: bucket name
-# return 0 for success, 1 for error
-abort_all_multipart_uploads() {
-  if [ $# -ne 1 ]; then
-    log 2 "'abort_all_multipart_uploads' requires bucket name"
-    return 1
-  fi
-  if ! list_multipart_uploads "$1"; then
-    log 2 "error listing multipart uploads"
-    return 1
-  fi
-  # shellcheck disable=SC2154
-  log 5 "UPLOADS: $uploads"
-  if ! upload_set=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -c '.Uploads[]' 2>&1); then
-    if [[ $upload_set == *"Cannot iterate over null"* ]]; then
-      return 0
-    else
-      log 2 "error getting upload set: $upload_set"
-      return 1
-    fi
-  fi
-  log 5 "UPLOAD SET: $upload_set"
-  for upload in $upload_set; do
-    log 5 "UPLOAD: $upload"
-    if ! upload_id=$(echo "$upload" | jq -r ".UploadId" 2>&1); then
-      log 2 "error getting upload ID: $upload_id"
-      return 1
-    fi
-    log 5 "upload ID: $upload_id"
-    if ! key=$(echo "$upload" | jq -r ".Key" 2>&1); then
-      log 2 "error getting key: $key"
-      return 1
-    fi
-    log 5 "Aborting multipart upload for key: $key, UploadId: $upload_id"
-    if ! abort_multipart_upload "$1" "$key" "$upload_id"; then
-      log 2 "error aborting multipart upload"
-      return 1
-    fi
-  done
-}
-
 create_upload_and_get_id_rest() {
   if [ $# -ne 2 ]; then
     log 2 "'create_upload_and_get_id_rest' requires bucket, key"
@@ -560,41 +498,6 @@ create_upload_and_get_id_rest() {
   return 0
 }
 
-create_abort_multipart_upload_rest() {
-  if [ $# -ne 2 ]; then
-    log 2 "'create_abort_upload_rest' requires bucket, key"
-    return 1
-  fi
-  if ! list_and_check_upload "$1" "$2"; then
-    log 2 "error listing multipart uploads before creation"
-    return 1
-  fi
-  log 5 "uploads before upload: $(cat "$TEST_FILE_FOLDER/uploads.txt")"
-  if ! create_upload_and_get_id_rest "$1" "$2"; then
-    log 2 "error creating upload"
-    return 1
-  fi
-  if ! list_and_check_upload "$1" "$2" "$upload_id"; then
-    log 2 "error listing multipart uploads after upload creation"
-    return 1
-  fi
-  log 5 "uploads after upload creation: $(cat "$TEST_FILE_FOLDER/uploads.txt")"
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/abort_multipart_upload.sh); then
-    log 2 "error aborting multipart upload: $result"
-    return 1
-  fi
-  if [ "$result" != "204" ]; then
-    log 2 "expected '204' response, actual was '$result' (error: $(cat "$TEST_FILE_FOLDER"/result.txt)"
-    return 1
-  fi
-  log 5 "final uploads: $(cat "$TEST_FILE_FOLDER/uploads.txt")"
-  if ! list_and_check_upload "$1" "$2"; then
-    log 2 "error listing multipart uploads after abort"
-    return 1
-  fi
-  return 0
-}
-
 multipart_upload_range_too_large() {
   if [ $# -ne 3 ]; then
     log 2 "'multipart_upload_range_too_large' requires bucket name, key, file location"
@@ -680,23 +583,6 @@ run_and_verify_multipart_upload_with_valid_range() {
   return 0
 }
 
-check_abort_access_denied() {
-  if [ $# -ne 5 ]; then
-    log 2 "'check_abort_access_denied' requires bucket, file, username, password"
-    return 1
-  fi
-  if abort_multipart_upload_with_user "$1" "$2" "$3" "$4" "$5"; then
-    log 2 "abort multipart upload succeeded despite lack of permissions"
-    return 1
-  fi
-  # shellcheck disable=SC2154
-  if [[ "$abort_multipart_upload_error" != *"AccessDenied"* ]]; then
-    log 2 "unexpected abort error:  $abort_multipart_upload_error"
-    return 1
-  fi
-  return 0
-}
-
 list_check_multipart_upload_key() {
   if [ $# -ne 4 ]; then
     log 2 "'list_check_multipart_upload_key' requires bucket, username, password, expected key"

tests/util/util_multipart_abort.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+check_abort_access_denied() {
+  if [ $# -ne 5 ]; then
+    log 2 "'check_abort_access_denied' requires bucket, file, username, password"
+    return 1
+  fi
+  if abort_multipart_upload_with_user "$1" "$2" "$3" "$4" "$5"; then
+    log 2 "abort multipart upload succeeded despite lack of permissions"
+    return 1
+  fi
+  # shellcheck disable=SC2154
+  if [[ "$abort_multipart_upload_error" != *"AccessDenied"* ]]; then
+    log 2 "unexpected abort error:  $abort_multipart_upload_error"
+    return 1
+  fi
+  return 0
+}
+
+create_abort_multipart_upload_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'create_abort_upload_rest' requires bucket, key"
+    return 1
+  fi
+  if ! list_and_check_upload "$1" "$2"; then
+    log 2 "error listing multipart uploads before creation"
+    return 1
+  fi
+  log 5 "uploads before upload: $(cat "$TEST_FILE_FOLDER/uploads.txt")"
+  if ! create_upload_and_get_id_rest "$1" "$2"; then
+    log 2 "error creating upload"
+    return 1
+  fi
+  if ! list_and_check_upload "$1" "$2" "$upload_id"; then
+    log 2 "error listing multipart uploads after upload creation"
+    return 1
+  fi
+  log 5 "uploads after upload creation: $(cat "$TEST_FILE_FOLDER/uploads.txt")"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/abort_multipart_upload.sh); then
+    log 2 "error aborting multipart upload: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    log 2 "expected '204' response, actual was '$result' (error: $(cat "$TEST_FILE_FOLDER"/result.txt)"
+    return 1
+  fi
+  log 5 "final uploads: $(cat "$TEST_FILE_FOLDER/uploads.txt")"
+  if ! list_and_check_upload "$1" "$2"; then
+    log 2 "error listing multipart uploads after abort"
+    return 1
+  fi
+  return 0
+}
+
+# param: bucket name
+# return 0 for success, 1 for error
+abort_all_multipart_uploads() {
+  if [ $# -ne 1 ]; then
+    log 2 "'abort_all_multipart_uploads' requires bucket name"
+    return 1
+  fi
+  if ! list_multipart_uploads "$1"; then
+    log 2 "error listing multipart uploads"
+    return 1
+  fi
+  # shellcheck disable=SC2154
+  log 5 "UPLOADS: $uploads"
+  if ! upload_set=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -c '.Uploads[]' 2>&1); then
+    if [[ $upload_set == *"Cannot iterate over null"* ]]; then
+      return 0
+    else
+      log 2 "error getting upload set: $upload_set"
+      return 1
+    fi
+  fi
+  log 5 "UPLOAD SET: $upload_set"
+  for upload in $upload_set; do
+    log 5 "UPLOAD: $upload"
+    if ! upload_id=$(echo "$upload" | jq -r ".UploadId" 2>&1); then
+      log 2 "error getting upload ID: $upload_id"
+      return 1
+    fi
+    log 5 "upload ID: $upload_id"
+    if ! key=$(echo "$upload" | jq -r ".Key" 2>&1); then
+      log 2 "error getting key: $key"
+      return 1
+    fi
+    log 5 "Aborting multipart upload for key: $key, UploadId: $upload_id"
+    if ! abort_multipart_upload "$1" "$key" "$upload_id"; then
+      log 2 "error aborting multipart upload"
+      return 1
+    fi
+  done
+}
+
+# run upload, then abort it
+# params:  bucket, key, local file location, number of parts to split into before uploading
+# return 0 for success, 1 for failure
+run_then_abort_multipart_upload() {
+  if [ $# -ne 4 ]; then
+    log 2 "run then abort multipart upload command missing bucket, key, file, and/or part count"
+    return 1
+  fi
+
+  if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then
+    log 2 "error performing pre-completion multipart upload"
+    return 1
+  fi
+
+  if ! abort_multipart_upload "$1" "$2" "$upload_id"; then
+    log 2 "error aborting multipart upload"
+    return 1
+  fi
+  return 0
+}

tests/util/util_policy.sh

@@ -310,7 +310,6 @@ put_and_check_policy_rest() {
     log 2 "unexpected response code, expected '200' or '204', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/result.txt"))"
     return 1
   fi
-  log 5 "response: $(cat "$TEST_FILE_FOLDER/result.txt")"
   if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then
     log 2 "error attempting to get bucket policy response: $result"
     return 1

tests/util/util_versioning.sh

@@ -49,23 +49,35 @@ delete_old_versions() {
   log 5 "version keys: ${version_keys[*]}"
   log 5 "version IDs: ${version_ids[*]}"
   for idx in "${!version_keys[@]}"; do
-    log 5 "idx: $idx"
-    log 5 "version ID: ${version_ids[$idx]}"
-    # shellcheck disable=SC2154
-    if [ "$lock_config_exists" == "true" ]; then
-      if ! delete_object_version_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
-        log 2 "error deleting object version"
-        return 1
-      fi
-    else
-      if ! delete_object_version "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
-        log 2 "error deleting object version"
-        return 1
-      fi
+    if ! delete_object_version_with_or_without_retention "$1"; then
+      log 2 "error deleting version with or without retention"
+      return 1
     fi
   done
 }
 
+delete_object_version_with_or_without_retention() {
+  if [ $# -ne 1 ]; then
+    log 2 "'delete_object_version_with_or_without_retention' requires bucket name"
+    return 1
+  fi
+  log 5 "idx: $idx"
+  log 5 "version ID: ${version_ids[$idx]}"
+  # shellcheck disable=SC2154
+  if [ "$lock_config_exists" == "true" ]; then
+    if ! delete_object_version_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
+      log 2 "error deleting object version"
+      return 1
+    fi
+  else
+    if ! delete_object_version "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
+      log 2 "error deleting object version"
+      return 1
+    fi
+  fi
+  return 0
+}
+
 parse_version_data() {
   if [ $# -ne 2 ]; then
     log 2 "'parse_version_data' requires raw data, element name"