Merge pull request #985 from versity/test/rest_put_acl_canned

Test/rest put acl canned
2026-01-04 11:03:57 +00:00 · 2024-12-16 09:18:15 -08:00
parent c17db864cd 10decc0d2c
commit 60700e3fa7
10 changed files with 275 additions and 41 deletions
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,5 +1,19 @@
 # Command-Line Tests

+## Table of Contents
+
+[Instructions - Running Locally](#instructions---running-locally)<br>
+[* Posix Backend](#posix-backend)<br>
+[* Static Bucket Mode](#static-bucket-mode)<br>
+[* S3 Backend](#s3-backend)<br>
+[* Direct Mode](#direct-mode)<br>
+[Instructions - Running With Docker](#instructions---running-with-docker)<br>
+[Instructions - Running With Docker-Compose](#instructions---running-with-docker-compose)<br>
+[Environment Parameters](#environment-parameters)<br>
+[* Secret](#secret)<br>
+[* Non-Secret](#non-secret)<br>
+[REST Scripts](#rest-scripts)<br>
+
 ## Instructions - Running Locally

 ### Posix Backend
@@ -85,8 +99,18 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up

 ## Environment Parameters

+### Secret
+
 **AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**:  identical to the same parameters in **s3**.

+**AWS_CANONICAL_ID**:  for direct mode, the canonical ID for the main user (owner)
+
+**ACL_AWS_CANONICAL_ID**:  for direct mode, the canonical ID for the user to test ACL changes and access by non-owners
+
+**ACL_AWS_ACCESS_KEY_ID**, **ACL_AWS_ACCESS_SECRET_KEY**:  for direct mode, the ID and key for the S3 user in the **ACL_AWS_CANONICAL_ID** account.
+
+### Non-Secret
+
 **VERSITY_EXE**:  location of the versity executable relative to test folder.

 **RUN_VERSITYGW**:  whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**.
@@ -134,3 +158,21 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 **VERSIONING_DIR**:  where to put gateway file versioning info.

 **COMMAND_LOG**:  where to store list of client commands, which if using will be reported during test failures.
+
+**TIME_LOG**:  optional log to show duration of individual tests
+
+**DIRECT_S3_ROOT_ACCOUNT_NAME**:  for direct mode, S3 username
+
+**DELETE_BUCKETS_AFTER_TEST**:  whether or not to delete buckets after individual tests, useful for debugging if the post-test bucket state needs to be checked
+
+## REST Scripts
+
+REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder.  To call a script, the following parameters are needed:
+* **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**, etc.
+* **AWS_ENDPOINT_URL** (default:  `https://localhost:7070`)
+* **OUTPUT_FILE**:  file where the command's response data is written
+* Any other parameters specified at the top of the script file, such as payloads and variables.  Sometimes, defaults are included.
+
+Upon success, the script will return a response code, and write the data to the **OUTPUT_FILE** location.
+
+Example:  `AWS_ACCESS_KEY_ID={id} AWS_SECRET_ACCESS_KEY={key} AWS_ENDPOINT_URL=https://s3.amazonaws.com OUTPUT_FILE=./output_file.xml ./tests/rest_scripts/list_buckets.sh`
--- a/tests/commands/put_bucket_acl.sh
+++ b/tests/commands/put_bucket_acl.sh
@@ -59,11 +59,11 @@ reset_bucket_acl() {
  fi
  # shellcheck disable=SC2154
  if [ "$DIRECT" != "true" ]; then
-    if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then
+    if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then
      log 2 "error resetting versitygw ACL"
      return 1
    fi
-  elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then
+  elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then
    log 2 "error resetting direct ACL"
    return 1
  fi
--- a/tests/logger.sh
+++ b/tests/logger.sh
@@ -55,7 +55,7 @@ log_mask() {
    return 1
  fi

-  log_message "$log_level" "${masked_args[*]}"
+  log_message "$log_level" "$masked_data"
 }

 mask_args() {
@@ -63,23 +63,38 @@ mask_args() {
    echo "'mask_args' requires string"
    return 1
  fi
-  IFS=' ' read -r -a array <<< "$1"
+  unmasked_array=()
+  masked_data=""
+  while IFS= read -r line; do
+    unmasked_array+=("$line")
+  done <<< "$1"

-  if ! mask_arg_array "${array[@]}"; then
-    echo "error masking arg array"
-    return 1
-  fi
+  # shellcheck disable=SC2068
+  first_line=true
+  for line in "${unmasked_array[@]}"; do
+    if ! mask_arg_array "$line"; then
+      echo "error masking arg array"
+      return 1
+    fi
+    if [ "$first_line" == "true" ]; then
+      masked_data="${masked_args[*]}"
+      first_line="false"
+    else
+      masked_data+=$(printf "\n%s" "${masked_args[*]}")
+    fi
+  done
 }

 mask_arg_array() {
-  masked_args=()  # Initialize an array to hold the masked arguments
  if [ $# -eq 0 ]; then
    echo "'mask_arg_array' requires parameters"
    return 1
  fi
  mask_next=false
  is_access=false
-  for arg in "$@"; do
+  masked_args=()  # Initialize an array to hold the masked arguments
+  # shellcheck disable=SC2068
+  for arg in $@; do
    if ! check_arg_for_mask "$arg"; then
      echo "error checking arg for mask"
      return 1
@@ -135,4 +150,5 @@ log_message() {
  if [[ -n "$TEST_LOG_FILE" ]]; then
    echo "$now $1 $2" >> "$TEST_LOG_FILE.tmp"
  fi
+  sync
 }
--- a/tests/rest_scripts/get_public_access_block.sh
+++ b/tests/rest_scripts/get_public_access_block.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="GET
+/$bucket_name
+publicAccessBlock=
+host:$host
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
+-H "\"x-amz-date: $current_date_time\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1
--- a/tests/rest_scripts/put_bucket_acl.sh
+++ b/tests/rest_scripts/put_bucket_acl.sh
@@ -22,8 +22,14 @@ source ./tests/rest_scripts/rest.sh
 bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2153
 acl_file="$ACL_FILE"
+# shellcheck disable=SC2153
+canned_acl="$CANNED_ACL"

-payload="$(cat "$acl_file")"
+if [ -n "$ACL_FILE" ]; then
+  payload="$(cat "$acl_file")"
+else
+  payload=""
+fi

 payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
@@ -32,20 +38,40 @@ canonical_request="PUT
 /$bucket_name
 acl=
 host:$host
-x-amz-content-sha256:$payload_hash
+"
+if [ -n "$CANNED_ACL" ]; then
+  canonical_request+="x-amz-acl:$canned_acl
+"
+fi
+canonical_request+="x-amz-content-sha256:$payload_hash
 x-amz-date:$current_date_time

-host;x-amz-content-sha256;x-amz-date
+"
+canonical_request+="host;"
+if [ -n "$CANNED_ACL" ]; then
+  canonical_request+="x-amz-acl;"
+fi
+canonical_request+="x-amz-content-sha256;x-amz-date
 $payload_hash"

 create_canonical_hash_sts_and_signature

-curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl="
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
-H "\"x-amz-content-sha256: $payload_hash\""
-H "\"x-amz-date: $current_date_time\""
-d "\"${payload//\"/\\\"}\""
-o "$OUTPUT_FILE")
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl=")
+if [ -n "$CANNED_ACL" ]; then
+  acl_header="x-amz-acl;"
+else
+  acl_header=""
+fi
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;${acl_header}x-amz-content-sha256;x-amz-date,Signature=$signature\"")
+if [ -n "$CANNED_ACL" ]; then
+  curl_command+=(-H "\"x-amz-acl: $canned_acl\"")
+fi
+curl_command+=(-H "\"x-amz-content-sha256: $payload_hash\""
+-H "\"x-amz-date: $current_date_time\"")
+if [ -n "$ACL_FILE" ]; then
+  curl_command+=(-d "\"${payload//\"/\\\"}\"")
+fi
+curl_command+=(-o "$OUTPUT_FILE")

 # shellcheck disable=SC2154
 eval "${curl_command[*]}" 2>&1
--- a/tests/rest_scripts/put_public_access_block.sh
+++ b/tests/rest_scripts/put_public_access_block.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+block_public_acls="${BLOCK_PUBLIC_ACLS:-TRUE}"
+ignore_public_acls="${IGNORE_PUBLIC_ACLS:-TRUE}"
+block_public_policy="${BLOCK_PUBLIC_POLICY:-TRUE}"
+restrict_public_buckets="${RESTRICT_PUBLIC_BUCKETS:-TRUE}"
+
+payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
+<PublicAccessBlockConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
+      <BlockPublicAcls>$block_public_acls</BlockPublicAcls>
+      <IgnorePublicAcls>$ignore_public_acls</IgnorePublicAcls>
+      <BlockPublicPolicy>$block_public_policy</BlockPublicPolicy>
+      <RestrictPublicBuckets>$restrict_public_buckets</RestrictPublicBuckets>
+</PublicAccessBlockConfiguration>"
+
+payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="PUT
+/$bucket_name
+publicAccessBlock=
+host:$host
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: $payload_hash\""
+-H "\"x-amz-date: $current_date_time\""
+-d "\"${payload//\"/\\\"}\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1
--- a/tests/setup.sh
+++ b/tests/setup.sh
@@ -86,6 +86,18 @@ teardown() {
      log 3 "unable to remove test file folder: $error"
    fi
  fi
+  stop_versity
+  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
+    end_time=$(date +%s)
+    total_time=$((end_time - start_time))
+    log 4 "Total test time: $total_time"
+    if [[ -n "$TIME_LOG" ]]; then
+      echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
+    fi
+  fi
+  if [[ -n "$COVERAGE_DB" ]]; then
+    record_result
+  fi
  if [[ "$BATS_TEST_COMPLETED" -ne 1 ]]; then
    if [[ -e "$COMMAND_LOG" ]]; then
      cat "$COMMAND_LOG"
@@ -108,16 +120,4 @@ teardown() {
      log 2 "error deleting temp log"
    fi
  fi
-  stop_versity
-  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
-    end_time=$(date +%s)
-    total_time=$((end_time - start_time))
-    log 4 "Total test time: $total_time"
-    if [[ -n "$TIME_LOG" ]]; then
-      echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
-    fi
-  fi
-  if [[ -n "$COVERAGE_DB" ]]; then
-    record_result
-  fi
 }
--- a/tests/test_rest.sh
+++ b/tests/test_rest.sh
@@ -38,6 +38,7 @@ source ./tests/util/util_list_parts.sh
 source ./tests/util/util_lock_config.sh
 source ./tests/util/util_ownership.sh
 source ./tests/util/util_policy.sh
+source ./tests/util/util_public_access_block.sh
 source ./tests/util/util_rest.sh
 source ./tests/util/util_tags.sh
 source ./tests/util/util_time.sh
@@ -244,9 +245,6 @@ export RUN_USERS=true
 }

@test "versioning - retrieve after delete" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/888"
-  fi
  test_file="test_file"

  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
@@ -269,9 +267,6 @@ export RUN_USERS=true
 }

@test "REST - legal hold, get without config" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/883"
-  fi
  test_file="test_file"

  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
@@ -490,4 +485,42 @@ export RUN_USERS=true

  run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file"
  assert_success
-}
+}
+
+@test "REST - put public-read canned acl" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test_file"
+  run create_test_files "$test_file"
+  assert_success
+
+  run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run create_versitygw_acl_user_or_get_direct_user "$USERNAME_ONE" "$PASSWORD_ONE"
+  assert_success
+  canonical_id=${lines[0]}
+  user_canonical_id=${lines[1]}
+  username=${lines[2]}
+  password=${lines[3]}
+
+  run list_objects_with_user_rest_verify_access_denied "$BUCKET_ONE_NAME" "$username" "$password"
+  assert_success
+
+  run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "$user_canonical_id" "READ" "$canonical_id"
+  assert_success
+
+  if [ "$DIRECT" == "true" ]; then
+    run allow_public_access "$BUCKET_ONE_NAME"
+    assert_success
+  fi
+  run put_acl_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/acl-file.txt"
+  assert_success
+
+  run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file"
+  assert_success
+}
--- a/tests/test_user_aws.sh
+++ b/tests/test_user_aws.sh
@@ -125,9 +125,6 @@ export RUN_USERS=true
 }

@test "test_admin_put_get_object" {
-  if [ "$RECREATE_BUCKETS" == "false" ]; then
-    skip "https://github.com/versity/versitygw/issues/888"
-  fi
  test_file="test_file"

  run setup_user_versitygw_or_direct "$USERNAME_ONE" "$PASSWORD_ONE" "admin" "$BUCKET_ONE_NAME"
--- a/tests/util/util_public_access_block.sh
+++ b/tests/util/util_public_access_block.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+allow_public_access() {
+  if [ $# -ne 1 ]; then
+    log 2 "'allow_public_access' requires bucket name"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/get_public_access_block.sh); then
+    log 2 "error getting public access block: $result"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" BLOCK_PUBLIC_ACLS="FALSE" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_public_access_block.sh); then
+    log 2 "error getting public access block: $result"
+    return 1
+  fi
+
+  return 0
+}