test: convert post-file-delete setup commands to REST

This commit is contained in:
Luke McCrone
2025-05-13 20:03:28 -03:00
parent 23cebcee2c
commit 18bcfebbab
18 changed files with 314 additions and 63 deletions

View File

@@ -27,8 +27,15 @@ send_command() {
fi
# shellcheck disable=SC2154
echo "${masked_args[*]}" >> "$COMMAND_LOG"
"$@"
return $?
fi
"$@"
local command_result=0
"$@" || command_result=$?
if [ "$command_result" -ne 0 ]; then
if [ "$1" == "curl" ]; then
echo ", curl response code: $command_result"
elif [ "$command_result" -ne 1 ]; then
echo " ($1 response code: $command_result)"
fi
fi
return $command_result
}

View File

@@ -44,13 +44,13 @@ copy_object() {
}
copy_object_empty() {
record-command "copy-object" "client:s3api"
record_command "copy-object" "client:s3api"
error=$(send_command aws --no-verify-ssl s3api copy-object 2>&1) || local result=$?
if [[ $result -eq 0 ]]; then
log 2 "copy object with empty parameters returned no error"
return 1
fi
if [[ $error != *"the following arguments are required: --bucket, --copy-source, --key" ]]; then
if [[ $error != *"the following arguments are required: --bucket, --copy-source, --key"* ]]; then
log 2 "copy object with no params returned mismatching error: $error"
return 1
fi

View File

@@ -50,4 +50,19 @@ delete_bucket() {
return 1
fi
return 0
}
delete_bucket_rest() {
if ! check_param_count "delete_bucket_rest" "bucket" 1 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_bucket.sh 2>&1); then
log 2 "error deleting bucket: $result"
return 1
fi
if [ "$result" != "204" ]; then
log 2 "expected '204', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt")"
return 1
fi
return 0
}

View File

@@ -45,6 +45,24 @@ delete_object() {
return 0
}
# shellcheck disable=SC2317
delete_object_rest() {
if [ $# -ne 2 ]; then
log 2 "'delete_object_rest' requires bucket name, object name"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
log 2 "error deleting object: $result"
return 1
fi
if [ "$result" != "204" ]; then
delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt")
log 2 "expected '204', was '$result' ($delete_object_error)"
return 1
fi
return 0
}
delete_object_bypass_retention() {
if ! check_param_count "delete_object_bypass_retention" "client, bucket, key, user, password" 5 $#; then
return 1

View File

@@ -37,6 +37,26 @@ get_bucket_ownership_controls() {
return 0
}
get_bucket_ownership_controls_rest() {
if ! check_param_count "get_bucket_ownership_controls_rest" "bucket" 1 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/ownershipControls.txt" ./tests/rest_scripts/get_bucket_ownership_controls.sh); then
log 2 "error getting bucket ownership controls: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "GetBucketOwnershipControls returned response code: $result, reply: $(cat "$TEST_FILE_FOLDER/ownershipControls.txt")"
return 1
fi
log 5 "controls: $(cat "$TEST_FILE_FOLDER/ownershipControls.txt")"
if ! rule=$(xmllint --xpath '//*[local-name()="ObjectOwnership"]/text()' "$TEST_FILE_FOLDER/ownershipControls.txt" 2>&1); then
log 2 "error getting ownership rule: $rule"
return 1
fi
echo "$rule"
}
get_object_ownership_rule() {
if [[ -n "$SKIP_BUCKET_OWNERSHIP_CONTROLS" ]]; then
log 5 "Skipping get bucket ownership controls"

View File

@@ -59,15 +59,15 @@ reset_bucket_acl() {
fi
# shellcheck disable=SC2154
if [ "$DIRECT" != "true" ]; then
if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then
if ! setup_acl "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then
log 2 "error resetting versitygw ACL"
return 1
fi
elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then
elif ! setup_acl "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then
log 2 "error resetting direct ACL"
return 1
fi
if ! put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$acl_file"; then
if ! put_bucket_acl_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$acl_file"; then
log 2 "error putting bucket acl (s3api)"
return 1
fi
@@ -113,3 +113,18 @@ put_bucket_canned_acl_with_user() {
fi
return 0
}
put_bucket_acl_rest() {
if ! check_param_count "put_bucket_acl_rest" "bucket, ACL file" 2 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" ACL_FILE="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_bucket_acl.sh); then
log 2 "error attempting to put bucket acl: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 5 "response returned code: $result (error: $(cat "$TEST_FILE_FOLDER/response.txt")"
return 1
fi
return 0
}

View File

@@ -27,15 +27,47 @@ put_object_lock_configuration() {
return 0
}
put_object_lock_configuration_disabled() {
if [[ $# -ne 1 ]]; then
log 2 "'put-object-lock-configuration' disable command requires bucket name"
remove_retention_policy_rest() {
if ! check_param_count "remove_retention_policy_rest" "bucket" 1 $#; then
return 1
fi
local config="{\"ObjectLockEnabled\": \"Enabled\"}"
if ! error=$(send_command aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_lock_configuration.sh 2>&1); then
log 2 "error putting object lock configuration: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
return 0
}
remove_retention_policy() {
if ! check_param_count "remove_retention_policy" "bucket" 1 $#; then
return 1
fi
if ! error=$(aws --no-verify-ssl s3api put-object-lock-configuration --bucket "$1" --object-lock-configuration "$config" 2>&1); then
log 2 "error putting object lock configuration: $error"
return 1
fi
return 0
}
put_object_lock_config_without_content_md5() {
if ! check_param_count "remove_retention_policy_rest" "bucket" 1 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OMIT_CONTENT_MD5="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_lock_configuration.sh 2>&1); then
log 2 "error putting object lock configuration: $result"
return 1
fi
if [ "$result" != "400" ]; then
log 2 "expected '400', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
if ! check_xml_error_contains "$TEST_FILE_FOLDER/result.txt" "InvalidRequest" "Content-MD5"; then
log 2 "error checking XML response"
return 1
fi
return 0
}

View File

@@ -39,10 +39,23 @@ log() {
return 0
}
# shellcheck disable=SC2317
log_with_stack_ref() {
if ! check_log_params "log_with_stack_ref" "level, message, stack reference" 3 $#; then
return 1
fi
if ! log_with_stack_ref "$1" "$2" 2; then
echo "error logging with stack ref"
return 1
fi
return 0
}
log_with_stack_ref() {
if [[ $# -ne 3 ]]; then
echo "log_with_stack_ref function requires level, message, stack reference"
return 1
fi
# shellcheck disable=SC2153
if [[ $1 -gt ${LOG_LEVEL_INT:=4} ]]; then
return 0

View File

@@ -0,0 +1,37 @@
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/rest_scripts/rest.sh
# shellcheck disable=SC2153
bucket_name="$BUCKET_NAME"
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
cr_data=("DELETE" "/$bucket_name" "")
cr_data+=("host:$host")
cr_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
build_canonical_request "${cr_data[@]}"
# shellcheck disable=SC2119
create_canonical_hash_sts_and_signature
curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "https://$host/$bucket_name"
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
curl_command+=("${header_fields[@]}")
curl_command+=(-o "$OUTPUT_FILE")
# shellcheck disable=SC2154
eval "${curl_command[*]}" 2>&1

View File

@@ -0,0 +1,71 @@
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/rest_scripts/rest.sh
# Fields
# shellcheck disable=SC2153
bucket_name="$BUCKET_NAME"
# shellcheck disable=SC2153
retention_rule="${RETENTION_RULE:=false}"
# shellcheck disable=SC2153
retention_days="$RETENTION_DAYS"
# shellcheck disable=SC2153
retention_mode="$RETENTION_MODE"
# shellcheck disable=SC2153
retention_years="$RETENTION_YEARS"
# shellcheck disable=SC2153
omit_content_md5="${OMIT_CONTENT_MD5:=false}"
payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<ObjectLockConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
<ObjectLockEnabled>Enabled</ObjectLockEnabled>"
if [ "$retention_rule" != "false" ]; then
payload+="<Rule>
<DefaultRetention>
<Days>$retention_days<Days>
<Mode>$retention_mode</Mode>
<Years>$retention_years</Years>
</DefaultRetention>
</Rule>"
fi
payload+="</ObjectLockConfiguration>"
payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
if [ "$omit_content_md5" == "false" ]; then
content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64)
fi
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
cr_data=("PUT" "/$bucket_name" "object-lock=")
if [ "$omit_content_md5" == "false" ]; then
cr_data+=("content-md5:$content_md5")
fi
cr_data+=("host:$host")
cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
build_canonical_request "${cr_data[@]}"
# shellcheck disable=SC2119
create_canonical_hash_sts_and_signature
curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?object-lock")
curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
curl_command+=("${header_fields[@]}")
curl_command+=(-d "\"${payload//\"/\\\"}\"" -o "$OUTPUT_FILE")
# shellcheck disable=SC2154
eval "${curl_command[*]}" 2>&1

View File

@@ -67,7 +67,7 @@ setup() {
export TEST_LOG_FILE
fi
if [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ]; then
if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && [ "$CREATE_STATIC_USERS_IF_NONEXISTENT" == "true" ]; then
if ! static_user_versitygw_setup; then
log 2 "error setting up static versitygw users"
return 1

View File

@@ -57,13 +57,13 @@ fi
username=${lines[2]}
password=${lines[3]}
run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "$user_canonical_id" "READ" "$canonical_id"
run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "CanonicalUser" "$user_canonical_id" "READ" "$canonical_id"
assert_success
run list_objects_with_user_rest_verify_access_denied "$BUCKET_ONE_NAME" "$username" "$password"
assert_success
run put_acl_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/acl-file.txt"
run put_bucket_acl_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/acl-file.txt"
assert_success
if [ "$DIRECT" == "true" ]; then
@@ -123,7 +123,7 @@ fi
username=${lines[2]}
password=${lines[3]}
run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "$user_canonical_id" "READD" "$canonical_id"
run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "CanonicalUser" "$user_canonical_id" "READD" "$canonical_id"
assert_success
if [ "$DIRECT" == "true" ]; then

View File

@@ -102,3 +102,40 @@ source ./tests/util/util_tags.sh
run check_object_lock_config_enabled_rest "$BUCKET_ONE_NAME"
assert_success
}
@test "REST - can set object lock enabled on existing buckets" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1300"
fi
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Enabled"
assert_success
# this enables object lock without a specific retention policy
run remove_retention_policy_rest "$BUCKET_ONE_NAME"
assert_success
}
@test "REST - cannot set object lock enabled without content-md5" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1301"
fi
run bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"
assert_success
# in static bucket config, bucket will still exist
if ! bucket_exists "$BUCKET_ONE_NAME"; then
run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
assert_success
fi
if [ "$DIRECT" == "true" ]; then
sleep 5
fi
# this enables object lock without a specific retention policy
run put_object_lock_config_without_content_md5 "$BUCKET_ONE_NAME"
assert_success
}

View File

@@ -270,21 +270,20 @@ get_and_check_acl_rest() {
}
setup_acl() {
if [ $# -ne 4 ]; then
log 2 "'setup_acl' requires acl file, grantee, permission, owner ID"
if ! check_param_count "setup_acl" "acl file, grantee type, grantee, permission, owner ID" 5 $#; then
return 1
fi
cat <<EOF > "$1"
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>$4</ID>
<ID>$5</ID>
</Owner>
<AccessControlList>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
<ID>$2</ID>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="$2">
<ID>$3</ID>
</Grantee>
<Permission>$3</Permission>
<Permission>$4</Permission>
</Grant>
</AccessControlList>
</AccessControlPolicy>
@@ -342,22 +341,6 @@ create_versitygw_acl_user_or_get_direct_user() {
fi
}
put_acl_rest() {
if [ $# -ne 2 ]; then
log 2 "'put_acl_rest' requires bucket name, ACL file"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" ACL_FILE="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_bucket_acl.sh); then
log 2 "error attempting to put bucket acl: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 5 "response returned code: $result (error: $(cat "$TEST_FILE_FOLDER/response.txt")"
return 1
fi
return 0
}
put_invalid_acl_rest_verify_failure() {
if [ $# -ne 2 ]; then
log 2 "'put_invalid_acl_rest_verify_failure' requires bucket name, ACL file"
@@ -405,16 +388,10 @@ check_ownership_rule_and_reset_acl() {
log 2 "'check_ownership_rule_and_reset_acl' requires bucket name"
return 1
fi
if ! get_bucket_ownership_controls "$1"; then
if ! object_ownership_rule=$(get_bucket_ownership_controls_rest "$1" 2>&1); then
log 2 "error getting bucket ownership controls"
return 1
fi
# shellcheck disable=SC2154
log 5 "ownership controls: $bucket_ownership_controls"
if ! object_ownership_rule=$(echo "$bucket_ownership_controls" | jq -r ".OwnershipControls.Rules[0].ObjectOwnership" 2>&1); then
log 2 "error getting object ownership rule: $object_ownership_rule"
return 1
fi
log 5 "ownership rule: $object_ownership_rule"
if [[ $object_ownership_rule != "BucketOwnerEnforced" ]] && ! reset_bucket_acl "$1"; then
log 2 "error resetting bucket ACL"

View File

@@ -66,8 +66,8 @@ clear_bucket_s3api() {
fi
# shellcheck disable=SC2154
if [[ $lock_config_exists == true ]] && ! put_object_lock_configuration_disabled "$1"; then
log 2 "error disabling object lock config"
if [[ $lock_config_exists == true ]] && ! remove_retention_policy_rest "$1"; then
log 2 "error removing bucket retention policy"
return 1
fi
@@ -90,7 +90,7 @@ delete_bucket_recursive_s3api() {
return 1
fi
if ! delete_bucket 's3api' "$1"; then
if ! delete_bucket_rest "$1"; then
log 2 "error deleting bucket"
return 1
fi

View File

@@ -1,20 +1,10 @@
#!/usr/bin/env bash
get_and_check_ownership_controls() {
if [ $# -ne 2 ]; then
log 2 "'get_and_check_ownership_controls' missing bucket name, expected result"
if ! check_param_count "get_and_check_ownership_controls" "bucket, expected result" 2 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/ownershipControls.txt" ./tests/rest_scripts/get_bucket_ownership_controls.sh); then
log 2 "error getting bucket ownership controls: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "GetBucketOwnershipControls returned response code: $result, reply: $(cat "$TEST_FILE_FOLDER/ownershipControls.txt")"
return 1
fi
log 5 "controls: $(cat "$TEST_FILE_FOLDER/ownershipControls.txt")"
if ! rule=$(xmllint --xpath '//*[local-name()="ObjectOwnership"]/text()' "$TEST_FILE_FOLDER/ownershipControls.txt" 2>&1); then
if ! rule=$(get_bucket_ownership_controls_rest "$1" 2>&1); then
log 2 "error getting ownership rule: $rule"
return 1
fi

View File

@@ -48,7 +48,7 @@ EOF
check_for_and_remove_worm_protection() {
log 6 "check_for_and_remove_worm_protection"
if ! check_param_count "check_for_and_remove_worm_protection" "bucket, key, error" 3 $#; then
return 1
return 2
fi
if [[ $3 == *"WORM"* ]]; then
@@ -114,3 +114,22 @@ retention_rest_without_request_body() {
fi
return 0
}
attempt_to_change_lock_config_without_content_md5() {
if ! check_param_count "attempt_to_change_lock_config_without_content_md5" "bucket" 1 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OMIT_CONTENT_MD5="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_lock_configuration.sh 2>&1); then
log 2 "error changing lock configuration: $result"
return 1
fi
if [ "$result" != "400" ]; then
log 2 "expected '400', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
if ! check_xml_error_contains "$TEST_FILE_FOLDER/result.txt" "InvalidRequest" "Content-MD5"; then
log 2 "error checking lock config error"
return 1
fi
return 0
}

View File

@@ -287,7 +287,7 @@ list_users() {
list_users_versitygw() {
log 6 "list_users_versitygw"
users=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users) || local list_result=$?
users=$(send_command "$VERSITY_EXE" admin --allow-insecure --access "$AWS_ACCESS_KEY_ID" --secret "$AWS_SECRET_ACCESS_KEY" --endpoint-url "$AWS_ENDPOINT_URL" list-users 2>&1) || local list_result=$?
if [[ $list_result -ne 0 ]]; then
log 2 "error listing users: $users"
return 1