mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-07-25 17:42:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2256 from versity/test/bucket_reset_cleanup

Browse Source 
test: bucket resetting code cleanup
This commit is contained in:
Ben McClelland
2026-07-20 21:59:07 -07:00
committed by GitHub
parent f377de8b76 4f3ae6c34d
commit 1e7d8d7848
18 changed files with 502 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/commands/get_bucket_location.sh
+6 -2
View File
@@ -86,9 +86,13 @@ get_bucket_location_rest() {
if ! check_param_count_v2 "bucket, callback" 2 $#; then
return 1
fi
if ! send_rest_go_command_callback "200" "$2" "-bucketName" "$1" "-method" "GET" "-query" "location=" "-awsRegion" "$AWS_REGION"; then
log 2 "error sending rest go command"
local bucket="$1" callback="$2"
local response
if ! response=$(send_rest_go_command_callback "200" "$callback" "-bucketName" "$bucket" "-method" "GET" "-query" "location=" 2>&1); then
log 2 "error sending rest go command: $response"
return 1
fi
echo "$response"
return 0
}
tests/commands/get_bucket_policy.sh
+125 -64
View File
@@ -19,23 +19,26 @@ get_bucket_policy() {
if ! check_param_count "get_bucket_policy" "command type, bucket" 2 $#; then
return 1
fi
local get_bucket_policy_result=0
if [[ $1 == 's3api' ]]; then
get_bucket_policy_s3api "$2" || get_bucket_policy_result=$?
elif [[ $1 == 's3cmd' ]]; then
get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$?
elif [[ $1 == 'mc' ]]; then
get_bucket_policy_mc "$2" || get_bucket_policy_result=$?
elif [ "$1" == 'rest' ]; then
get_bucket_policy_rest "$2" || get_bucket_policy_result=$?
local command_type="$1" bucket="$2"
local response get_bucket_policy_result=0
if [[ $command_type == 's3api' ]]; then
response=$(get_bucket_policy_s3api "$bucket" 2>&1) || get_bucket_policy_result=$?
elif [[ $command_type == 's3cmd' ]]; then
response=$(get_bucket_policy_s3cmd "$bucket" 2>&1) || get_bucket_policy_result=$?
elif [[ $command_type == 'mc' ]]; then
response=$(get_bucket_policy_mc "$bucket" 2>&1) || get_bucket_policy_result=$?
elif [ "$command_type" == 'rest' ]; then
response=$(get_bucket_policy_rest "$bucket" 2>&1) || get_bucket_policy_result=$?
else
log 2 "command 'get bucket policy' not implemented for '$1'"
log 2 "command 'get bucket policy' not implemented for '$command_type'"
return 1
fi
if [[ $get_bucket_policy_result -ne 0 ]]; then
log 2 "error getting policy: $bucket_policy"
log 2 "error getting policy: $response"
return 1
fi
echo "$response"
return 0
}
@@ -44,19 +47,20 @@ get_bucket_policy_s3api() {
if ! check_param_count "get_bucket_policy_s3api" "bucket" 1 $#; then
return 1
fi
policy_json=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
log 5 "$policy_json"
if [[ $get_result -ne 0 ]]; then
if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then
local bucket="$1"
local response policy_json bucket_policy
if ! response=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$bucket" 2>&1); then
if [[ "$response" == *"(NoSuchBucketPolicy)"* ]]; then
bucket_policy=
else
log 2 "error getting policy: $policy_json"
log 2 "error getting policy: $response"
return 1
fi
else
bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
fi
policy_json=$(echo "$response" | grep -v "InsecureRequestWarning")
bucket_policy=$(echo "$policy_json" | jq -r '.Policy')
echo "$bucket_policy"
return 0
}
@@ -82,81 +86,138 @@ get_bucket_policy_s3cmd() {
if ! check_param_count "get_bucket_policy_s3cmd" "bucket" 1 $#; then
return 1
fi
local bucket="$1"
local response s3cmd_response bucket_policy
if ! info=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --region "$AWS_REGION" info "s3://$1" 2>&1); then
log 2 "error getting bucket policy: $info"
if ! response=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --region "$AWS_REGION" info "s3://$bucket" 2>&1); then
log 2 "error getting bucket policy: $response"
return 1
fi
s3cmd_response="$response"
log 5 "policy info: $info"
bucket_policy=""
policy_brackets=false
log 5 "policy info: $response"
# NOTE: versitygw sends policies back in multiple lines here, direct in single line
while IFS= read -r line; do
if check_and_load_policy_info; then
break
fi
done <<< "$info"
log 5 "bucket policy: $bucket_policy"
if ! response=$(parse_s3cmd_policy "$s3cmd_response"); then
log 2 "error parsing s3cmd policy: $response"
return 1
fi
log 5 "bucket policy: $response"
echo "$response"
return 0
}
parse_s3cmd_policy() {
if ! check_param_count_v2 "response data" 1 $#; then
return 1
fi
local response_data="$1"
local line policy=""
while IFS= read -r line; do
if [[ -z $policy ]]; then
[[ $line =~ ^[[:space:]]*Policy:[[:space:]]*(.*)$ ]] || continue
policy=${BASH_REMATCH[1]}
else
policy+=$'\n'"$line"
fi
if [ "$policy" == "none" ]; then
echo ""
return 0
elif jq -e . >/dev/null 2>&1 <<<"$policy"; then
printf '%s\n' "$policy"
return 0
fi
done <<< "$response_data"
log 2 "policy data not found (data: '$response_data')"
return 1
}
get_bucket_policy_rest() {
if ! check_param_count "get_bucket_policy_rest" "bucket" 1 $#; then
if ! check_param_count_ge_le "bucket, region (optional)" 1 2 $#; then
return 1
fi
log 5 "aws region: $AWS_REGION"
if ! get_bucket_policy_rest_expect_code "$1" "200"; then
log 2 "error getting REST bucket policy"
local bucket="$1" region="$2"
local response
log 5 "aws region: $2"
if ! response=$(get_bucket_policy_rest_expect_code "$bucket" "200" "$region" 2>&1); then
log 2 "error getting REST bucket policy: $response"
return 1
fi
echo "$response"
return 0
}
get_bucket_policy_rest_go() {
if ! check_param_count_gt "bucket, params (optional)" 1 $#; then
return 1
fi
local bucket="$1" params=("${@:2}")
local response
if ! response=$(send_rest_go_command "200" "-bucketName" "$bucket" "-query" "policy" "${params[@]}" 2>&1); then
log 2 "error getting bucket policy: $response"
return 1
fi
echo "$response"
return 0
}
get_bucket_policy_rest_expect_code() {
if ! check_param_count "get_bucket_policy_rest_expect_code" "bucket, code" 2 $#; then
if ! check_param_count_ge_le "bucket, code, region (optional)" 2 3 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then
log 2 "error attempting to get bucket policy response: $result"
return 1
fi
if [ "$result" != "$2" ]; then
log 2 "unexpected response code, expected '$2', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/policy.txt"))"
return 1
fi
bucket_policy="$(cat "$TEST_FILE_FOLDER/policy.txt")"
}
local bucket_name="$1" expected_response_code="$2" region="$3"
local region_string response file_name
# return 0 for no policy, single-line policy, or loading complete, 1 for still searching or loading
check_and_load_policy_info() {
if [[ $policy_brackets == false ]]; then
if search_for_first_policy_line_or_full_policy; then
return 0
fi
if [ "$region" != "" ]; then
region_string="AWS_REGION=$3"
else
bucket_policy+=$line
if [[ $line == "}" ]]; then
return 0
fi
region_string="AWS_REGION=$AWS_REGION"
fi
return 1
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! response=$(env COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket_name" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" "$region_string" ./tests/rest_scripts/get_bucket_policy.sh 2>&1); then
log 2 "error attempting to get bucket policy response: $response"
return 1
fi
if [ "$response" != "$expected_response_code" ]; then
log 2 "unexpected response code, expected '$expected_response_code', actual '$response' (reply: '$(cat "$TEST_FILE_FOLDER/$file_name")')"
return 1
fi
bucket_policy="$(cat "$TEST_FILE_FOLDER/$file_name")"
echo "$bucket_policy"
return 0
}
# return 0 for empty or single-line policy, 1 for other cases
# return 0 for empty or single-line policy, 1 for not found or in progress, 2 for error
search_for_first_policy_line_or_full_policy() {
policy_line=$(echo "$line" | grep 'Policy: ')
if [[ $policy_line != "" ]]; then
if [[ $policy_line != *'{'* ]]; then
if ! check_param_count_v2 "line" 1 $#; then
return 2
fi
local line="$1"
local policy_line
if grep 'Policy: ' "$line" >/dev/null; then
if [[ $line != *'{'* ]]; then
echo ""
return 0
fi
if [[ $policy_line == *'}'* ]]; then
if [[ $line == *'}'* ]]; then
log 5 "policy on single line"
bucket_policy=${policy_line//Policy:/}
policy_line=${line//Policy:/}
echo "$policy_line"
return 0
else
policy_brackets=true
bucket_policy+="{"
echo "{"
fi
fi
return 1
tests/commands/get_object_lock_configuration.sh
+15
View File
@@ -65,4 +65,19 @@ get_object_lock_configuration_rest() {
echo "$response_code" "$TEST_FILE_FOLDER/$output_file"
return $return_code
}
get_object_lock_configuration_rest_go() {
if ! check_param_count_gt "bucket, callback, additional params" 2 $#; then
return 1
fi
local bucket="$1" callback="$2"
local response
if ! response=$(send_rest_go_command_callback "200" "$callback" "-query" "object-lock" "-bucketName" "$bucket" "${@:3}" 2>&1); then
log 2 "error getting object lock configuration: $response"
return 1
fi
echo "$response"
return 0
}
tests/commands/list_object_versions.sh
+17 -10
View File
@@ -18,17 +18,21 @@ list_object_versions() {
if ! check_param_count "list_object_versions" "client, bucket name" 2 $#; then
return 1
fi
local list_result=0
if [ "$1" == "rest" ]; then
list_object_versions_rest "$2" || list_result=$?
local client="$1" bucket="$2"
local list_result=0 response response_with_warning versions
if [ "$client" == "rest" ]; then
response=$(list_object_versions_rest "$bucket" 2>&1) || list_result=$?
else
versions=$(send_command aws --no-verify-ssl s3api list-object-versions --bucket "$2" 2>&1) || list_result=$?
response_with_warning=$(send_command aws --no-verify-ssl s3api list-object-versions --bucket "$2" 2>&1) || list_result=$?
response=$(echo "$response_with_warning" | grep -v "InsecureRequestWarning")
fi
if [[ $list_result -ne 0 ]]; then
log 2 "error listing object versions: $versions"
return 1
fi
versions=$(echo "$versions" | grep -v "InsecureRequestWarning")
versions="$response"
echo "$versions"
return 0
}
@@ -51,18 +55,21 @@ list_object_versions_rest() {
log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/$file_name"))"
return 1
fi
versions=$(cat "$TEST_FILE_FOLDER/$file_name")
echo "$TEST_FILE_FOLDER/$file_name"
cat "$TEST_FILE_FOLDER/$file_name"
return 0
}
list_object_versions_rest_v2() {
if ! check_param_count_v2 "bucket name, callback" 2 $#; then
if ! check_param_count_gt "bucket name, callback, params (optional)" 2 $#; then
return 1
fi
if ! send_rest_go_command_callback "200" "$2" "-bucketName" "$1" "-query" "versions="; then
log 2 "error sending REST list object versions command"
local bucket="$1" callback="$2" params=("${@:3}")
local response
if ! response=$(send_rest_go_command_callback "200" "$callback" "-bucketName" "$bucket" "-query" "versions=" "${params[@]}" 2>&1); then
log 2 "error sending REST list object versions command: $response"
return 1
fi
echo "$response"
return 0
}
tests/commands/list_objects.sh
+15
View File
@@ -151,6 +151,21 @@ list_objects_rest() {
return 0
}
list_objects_rest_go() {
if ! check_param_count_gt "bucket, callback, params (optional)" 2 $#; then
return 1
fi
local bucket="$1" callback="$2" params=(${@:3})
local response
if ! response=$(send_rest_go_command_callback "200" "$callback" "-bucketName" "$bucket" "${params[@]}" 2>&1); then
log 2 "error sending ListObjects REST command: $response"
return 1
fi
echo "$response"
return 0
}
list_objects_rest_expect_error() {
if ! check_param_count_v2 "bucket name, env vars, response code, error, message" 5 $#; then
return 1
tests/drivers/delete_bucket/delete_bucket_rest.sh
+7 -5
View File
@@ -132,19 +132,21 @@ delete_bucket_recursive() {
if ! check_param_count "delete_bucket_recursive_s3api" "bucket" 1 $#; then
return 1
fi
local bucket="$1"
local response region endpoint
if ! location=$(get_bucket_location_rest "$1" "parse_bucket_location" 2>&1); then
log 2 "error getting bucket location: $location"
if ! response=$(get_bucket_location_and_endpoint "$bucket" 2>&1); then
log 2 "error getting bucket location and endpoint: $response"
return 1
fi
log 5 "location: $location"
read -r region endpoint <<< "$response"
if ! reset_bucket "$1"; then
if ! reset_bucket "$bucket" "$region" "$endpoint"; then
log 2 "error clearing bucket (s3api)"
return 1
fi
if ! delete_bucket_rest "$1"; then
if ! AWS_REGION="$region" AWS_ENDPOINT_URL="$endpoint" delete_bucket_rest "$bucket"; then
log 2 "error deleting bucket"
return 1
fi
tests/drivers/delete_object/delete_object_rest.sh
+64 -22
View File
@@ -20,12 +20,13 @@ source ./tests/drivers/xml.sh
source ./tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh
list_and_delete_objects() {
log 6 "list_and_delete_objects: '$1'"
if ! check_param_count_v2 "bucket" 1 $#; then
if ! check_param_count_v2 "bucket, lock config exists" 2 $#; then
return 1
fi
local bucket="$1" lock_config_exists="$2"
local response
if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then
if ! response=$(list_objects_rest_go "$bucket" "parse_objects_list_rest" 2>&1); then
log 2 "error getting object list: $response"
return 1
fi
@@ -35,44 +36,85 @@ list_and_delete_objects() {
if [ "$object" == "" ]; then
break
fi
if ! clear_object_in_bucket "$1" "$object"; then
if ! clear_object_in_bucket "$bucket" "$object"; then
log 2 "error deleting object $object"
return 1
fi
done
if ! delete_old_versions_base64 "$1"; then
if ! delete_old_versions "$bucket" "$lock_config_exists"; then
log 2 "error deleting old version"
return 1
fi
return 0
}
delete_old_versions_base64() {
if ! check_param_count "delete_old_versions" "bucket" 1 $#; then
delete_old_versions() {
if ! check_param_count_v2 "bucket, lock config exists (true or false)" 2 $#; then
return 1
fi
if ! list_object_versions "rest" "$1"; then
log 2 "error listing object versions"
return 1
fi
# shellcheck disable=SC2154
log 5 "versions: $versions"
local bucket="$1" lock_config_exists="$2"
local response versions_xml
if ! parse_base64_versions_rest; then
log 2 "error parsing version data"
if ! response=$(list_object_versions_rest_v2 "$bucket" "get_xml_versions_data" 2>&1); then
log 2 "error listing object versions: $response"
return 1
fi
versions_xml="$response"
# shellcheck disable=SC2154
log 5 "base64 versions: ${base64_pairs[*]}"
for pair in "${base64_pairs[@]}"; do
log 5 "pair: $pair"
if ! delete_object_version_with_or_without_retention_base64 "$1" "$pair"; then
log 2 "error deleting version with or without retention"
if ! response=$(get_and_delete_old_versions_from_xml "$versions_xml" "Version" "$bucket" "$lock_config_exists" 2>&1); then
log 2 "error deleting Versions: $response"
return 1
fi
if ! response=$(get_and_delete_old_versions_from_xml "$versions_xml" "DeleteMarker" "$bucket" "$lock_config_exists" 2>&1); then
log 2 "error deleting DeleteMarkers: $response"
return 1
fi
return 0
}
get_and_delete_old_versions_from_xml() {
if ! check_param_count_v2 "version XML, 'Version' or 'DeleteMarker', bucket, lock config exists" $# 4; then
return 1
fi
local version_xml="$1" version_or_delete_marker="$2" bucket="$3" lock_config_exists="$4"
while IFS= read -r key && IFS= read -r version_id; do
if ! delete_single_version_or_delete_marker "$bucket" "$lock_config_exists" "$key" "$version_id"; then
log 2 "error deleting key '$key', version ID '$version_id'"
return 1
fi
done
done < <(xmlstarlet sel -t \
-m '//*[local-name()='"\"$version_or_delete_marker\""']' \
-v '*[local-name()="Key"]' -n \
-v '*[local-name()="VersionId"]' -n \
<<<"$version_xml" | xmlstarlet unesc)
return 0
}
delete_single_version_or_delete_marker() {
if ! check_param_count_v2 "bucket, lock config exists, key, version ID" 4 $#; then
return 1
fi
local bucket="$1" lock_config="$2" key="$3" version_id="$4"
if [ "$lock_config" == "true" ]; then
if ! check_remove_legal_hold_versions "$bucket" "$key" "$version_id"; then
log 2 "error checking, removing legal hold versions"
return 1
fi
if ! delete_object_version_rest_bypass_retention "$bucket" "$key" "$version_id"; then
log 2 "error deleting object version, bypassing retention"
return 1
fi
else
if ! delete_object_version_rest "$bucket" "$key" "$version_id"; then
log 2 "error deleting object version"
return 1
fi
fi
log 5 "successfully deleted version with key '$key', id '$version_id'"
return 0
}
delete_object_version_with_or_without_retention_base64() {
tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh
+3 -2
View File
@@ -30,12 +30,13 @@ check_legal_hold_without_lock_enabled() {
}
check_remove_legal_hold_versions() {
if ! check_param_count "check_remove_legal_hold_versions" "bucket, key, version ID" 3 $#; then
if ! check_param_count_gt "bucket, key, version ID" 3 $#; then
return 1
fi
local bucket="$1" key="$2" version_id="$3"
local response legal_hold_data
if ! response=$(get_object_legal_hold_rest_version_id "$1" "$2" "$3" 2>&1); then
if ! response=$(get_object_legal_hold_rest_version_id "$bucket" "$key" "$version_id" 2>&1); then
# shellcheck disable=SC2154
log 5 "legal hold: $response"
if [[ "$response" != *"MethodNotAllowed"* ]] && [[ "$response" != *"NoSuchObjectLockConfiguration"* ]]; then
tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh
+19
View File
@@ -63,3 +63,22 @@ check_object_lock_config_enabled_rest() {
fi
return 0
}
check_object_lock_config_go() {
if ! check_param_count_gt "bucket, additional params (optional)" 1 $#; then
return 1
fi
local bucket="$1" params=("${@:2}")
local response
if ! response=$(get_object_lock_configuration_rest_go "$bucket" "" "${params[@]}" 2>&1); then
if [[ "$response" == *"HTTP/1.1 404 Not Found"* ]]; then
echo "false"
return 0
fi
log 2 "error checking object lock configuration: $response"
return 1
fi
echo "true"
return 0
}
tests/drivers/head_bucket/head_bucket_rest.sh
+73
View File
@@ -38,3 +38,76 @@ check_bucket_existence_callback() {
echo "error checking if bucket exists (data: $2)"
return 2
}
get_endpoint() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
local data_file="$1"
local response endpoint
if ! response=$(get_element_text "$data_file" "Error" "Endpoint" 2>&1); then
log 2 "error getting endpoint: $response"
return 1
fi
endpoint="$response"
log 5 "endpoint: $endpoint"
echo "$endpoint"
return 0
}
head_bucket_get_endpoint() {
if ! check_param_count_gt "bucket name, params (optional)" 1 $#; then
return 1
fi
local bucket="$1"
local response endpoint
if ! response=$(send_rest_go_command_expect_error_callback "301" "PermanentRedirect" "must be addressed" "get_endpoint" "-bucketName" "$bucket" "${@:2}" 2>&1); then
if [[ "$response" == *"HTTP/1.1 200 OK"* ]]; then
echo "$AWS_ENDPOINT_URL"
return 0
fi
log 2 "error getting response: $response"
return 1
fi
log 5 "response: $response"
endpoint="$response"
log 5 "returned endpoint: $endpoint"
echo "https://${endpoint}"
return 0
}
get_bucket_location_and_endpoint() {
if ! check_param_count_v2 "bucket" 1 $#; then
return 1
fi
local bucket="$1"
local response region endpoint_url
if ! response=$(get_bucket_location_rest "$1" "parse_bucket_location" 2>&1); then
log 2 "error getting bucket location: $response"
return 1
fi
region="$response"
if [ "$region" == "" ]; then
region="us-east-1"
fi
if [ "$DIRECT" != "true" ]; then
echo "$region $AWS_ENDPOINT_URL"
return 0
fi
if ! response=$(AWS_REGION="$region" head_bucket_get_endpoint "$bucket" 2>&1); then
log 2 "error getting bucket region and endpoint: $response"
return 1
fi
if [ "$response" != "$AWS_ENDPOINT_URL" ]; then
endpoint_url="https://s3.${region}.amazonaws.com"
else
endpoint_url="$AWS_ENDPOINT_URL"
fi
echo "$region" "$endpoint_url"
return 0
}
tests/drivers/head_object/head_object_rest.sh
+17 -10
View File
@@ -256,32 +256,39 @@ check_header_partial_content_response() {
}
get_delete_marker_and_verify_405() {
if ! check_param_count "get_delete_marker_and_verify_405" "bucket, file name" 2 $#; then
if ! check_param_count_v2 "bucket, key" 2 $#; then
return 1
fi
if ! response=$(list_object_versions_rest "$1" 2>&1); then
local bucket="$1" key="$2"
local response versions_data version_id file_name response_code
if ! response=$(list_object_versions_rest "$bucket" 2>&1); then
log 2 "error listing REST object versions"
return 1
fi
versions_file="$response"
log 5 "versions: $(cat "$versions_file")"
versions_data="$response"
log 5 "versions: $versions_data"
if ! version_id=$(xmllint --xpath "//*[local-name()=\"DeleteMarker\"]/*[local-name()=\"VersionId\"]/text()" "$versions_file" 2>&1); then
if ! response=$(xmllint --xpath "//*[local-name()=\"DeleteMarker\"]/*[local-name()=\"VersionId\"]/text()" - <<< "$versions_data" 2>&1); then
log 2 "error getting XML value: $version_id"
return 1
fi
log 5 "xml val: $version_id"
version_id="$response"
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/$response" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then
log 2 "error getting result: $result"
file_name="$response"
if ! response=$(OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then
log 2 "error getting result: $response"
return 1
fi
if [ "$result" != "405" ]; then
log 2 "expected '405', was '$result' ($(cat "$TEST_FILE_FOLDER/$response"))"
response_code="$response"
if [ "$response_code" != "405" ]; then
log 2 "expected '405', was '$response_code' ($(cat "$TEST_FILE_FOLDER/$file_name"))"
return 1
fi
return 0
tests/drivers/list_object_versions/list_object_versions_rest.sh
+16 -1
View File
@@ -272,4 +272,19 @@ list_object_versions_with_prefix_and_delimiter_check_results() {
return 1
fi
return 0
}
}
get_xml_versions_data() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
local data_file="$1"
local response
if ! response=$(get_element "$data_file" "ListVersionsResult" 2>&1); then
log 2 "error getting ListVersionsResult: $response"
return 1
fi
printf '%s\n' "$response"
return 0
}
tests/drivers/list_objects/list_objects_rest.sh
+7 -6
View File
@@ -356,17 +356,18 @@ parse_objects_list_rest() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
local data_file="$1"
local response object_list object unescaped_object
local response
if ! response=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then
if [[ "$response" == *"XPath set is empty"* ]]; then
if ! response=$(get_element_text "$data_file" "Key" 2>&1); then
if [[ "$response" == *"element matching"*"doesn't exist"* ]]; then
return 0
fi
log 2 "error getting object list: $response"
log 2 "error getting object keys: $response"
return 1
else
object_list="$response"
fi
object_list="$response"
log 5 "object list: '$object_list'"
while IFS= read -r object; do
log 5 "parsed key: '$object'"
tests/drivers/rest.sh
+1
View File
@@ -309,6 +309,7 @@ send_rest_go_command_expect_error_callback() {
log 2 "callback error: $response"
return 1
fi
echo "$response"
return 0
}
tests/drivers/user.sh
+35 -13
View File
@@ -119,61 +119,83 @@ setup_bucket_file_and_user_v2() {
return 0
}
reset_bucket() {
if ! check_param_count "reset_bucket" "bucket" 1 $#; then
perform_reset() {
if ! check_param_count_v2 "bucket" 1 $#; then
return 1
fi
log 6 "reset bucket '$1'"
local bucket="$1"
local response lock_config_exists
if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$1"; then
if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$bucket"; then
log 3 "error logging bucket policy"
fi
if ! response=$(check_object_lock_config "$1" 2>&1); then
if ! response=$(check_object_lock_config_go "$bucket" 2>&1); then
log 2 "error checking object lock config: $response"
return 1
fi
lock_config_exists="$response"
if [[ "$DIRECT" != "true" ]] && ! add_governance_bypass_policy "$1"; then
if [[ "$DIRECT" != "true" ]] && ! add_governance_bypass_policy "$bucket"; then
log 2 "error adding governance bypass policy"
return 1
fi
if ! list_and_delete_objects "$1"; then
if ! list_and_delete_objects "$bucket" "$lock_config_exists"; then
log 2 "error listing and deleting objects"
return 1
fi
if ! abort_all_multipart_uploads_rest "$1"; then
if ! abort_all_multipart_uploads_rest "$bucket"; then
log 2 "error aborting all multipart uploads"
return 1
fi
if [ "$SKIP_ACL_TESTING" != "true" ] && ! check_ownership_rule_and_reset_acl "$1"; then
if [ "$SKIP_ACL_TESTING" != "true" ] && ! check_ownership_rule_and_reset_acl "$bucket"; then
log 2 "error checking ownership rule and resetting acl"
return 1
fi
if ! delete_bucket_policy_rest "$1"; then
if ! delete_bucket_policy_rest "$bucket"; then
log 2 "error deleting bucket policy"
return 1
fi
# shellcheck disable=SC2154
if [[ $lock_config_exists == true ]] && ! remove_retention_policy_rest "$1"; then
if [[ $lock_config_exists == true ]] && ! remove_retention_policy_rest "$bucket"; then
log 2 "error removing bucket retention policy"
return 1
fi
if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID"; then
if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$bucket" "$AWS_ACCESS_KEY_ID"; then
log 2 "error changing bucket owner back to root"
return 1
fi
return 0
}
reset_bucket() {
if ! check_param_count_ge_le "bucket, region (optional), endpoint (optional)" 1 3 $#; then
return 1
fi
local bucket="$1" region="$2" endpoint="$3"
local response
if [ -z "$region" ] || [ -z "$endpoint" ]; then
if ! response=$(get_bucket_location_and_endpoint "$bucket" 2>&1); then
log 2 "error getting bucket region and endpoint: $response"
return 1
fi
read -r region endpoint <<< "$response"
fi
if ! response=$(AWS_REGION="$region" AWS_ENDPOINT_URL="$endpoint" perform_reset "$bucket" 2>&1); then
log 2 "error performing bucket reset: $response"
return 1
fi
return 0
}
get_user_id() {
if [ "$DIRECT" == "true" ]; then
if [ "$DIRECT_AWS_USER_ID" == "" ]; then
@@ -185,4 +207,4 @@ get_user_id() {
fi
echo "$AWS_ACCESS_KEY_ID"
return 0
}
}
tests/test_rest_delete_bucket.sh
+1 -1
View File
@@ -87,7 +87,7 @@ source ./tests/drivers/put_object/put_object_rest.sh
run delete_bucket_rest_expect_error "$bucket_name" "" "409" "BucketNotEmpty" "is not empty"
assert_success
run delete_old_versions_base64 "$bucket_name"
run delete_old_versions "$bucket_name" "false"
assert_success
run delete_bucket_rest "$bucket_name"
tests/util/util_policy.sh
+46 -20
View File
@@ -190,14 +190,18 @@ get_and_check_policy() {
if ! check_param_count "get_and_check_policy" "client, bucket, expected effect, principal, action, resource" 6 $#; then
return 1
fi
if ! get_bucket_policy "$1" "$2"; then
log 2 "error getting bucket policy after setting"
local client="$1" bucket="$2" expected_effect="$3" expected_principal="$4" expected_action="$5" expected_resource="$6"
local response policy
if ! response=$(get_bucket_policy "$client" "$bucket" 2>&1); then
log 2 "error getting bucket policy after setting: $response"
return 1
fi
policy="$response"
# shellcheck disable=SC2154
log 5 "POLICY: $bucket_policy"
if ! check_policy "$bucket_policy" "$3" "$4" "$5" "$6"; then
log 5 "POLICY: $policy"
if ! check_policy "$policy" "$expected_effect" "$expected_principal" "$expected_action" "$expected_resource"; then
log 2 "error checking policy"
return 1
fi
@@ -304,28 +308,40 @@ get_and_compare_policy_with_file() {
}
put_and_check_policy_rest() {
if ! check_param_count "put_and_check_policy_rest" "bucket, policy file, effect, principal, action, resource" 6 $#; then
if ! check_param_count_v2 "bucket, policy file, effect, principal ID, action, resource" 6 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" POLICY_FILE="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_bucket_policy.sh); then
log 2 "error putting policy: $result"
local bucket="$1" policy_file="$2" expected_effect="$3" principal_id="$4" expected_action="$5" expected_resource="$6"
local response file_name response_code policy expected_principal
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
log 5 "response code: $result"
if [[ ( "$result" != "204" ) && ( "$result" != "200" ) ]]; then
log 2 "unexpected response code, expected '200' or '204', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/result.txt"))"
file_name="$response"
if ! response=$(env COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" POLICY_FILE="$policy_file" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/put_bucket_policy.sh 2>&1); then
log 2 "error putting policy: $response"
return 1
fi
if ! get_bucket_policy_rest "$1"; then
log 2 "error attempting to get bucket policy response: $result"
response_code="$response"
if [[ ( "$response_code" != "204" ) && ( "$response_code" != "200" ) ]]; then
log 2 "unexpected response code, expected '200' or '204', actual '$response_code' (reply: '$(cat "$TEST_FILE_FOLDER/$file_name")')"
return 1
fi
if ! response=$(get_bucket_policy_rest "$bucket" 2>&1); then
log 2 "error attempting to get bucket policy response: $response"
return 1
fi
policy="$response"
if [ "$DIRECT" == "true" ]; then
principal="arn:aws:iam::$DIRECT_AWS_USER_ID:user/$4"
expected_principal="arn:aws:iam::$DIRECT_AWS_USER_ID:user/$principal_id"
else
principal="$4"
expected_principal="$principal_id"
fi
if ! check_policy "$bucket_policy" "$3" "$principal" "$5" "$6"; then
if ! check_policy "$policy" "$expected_effect" "$expected_principal" "$expected_action" "$expected_resource"; then
log 2 "policies not equal"
return 1
fi
@@ -334,13 +350,23 @@ put_and_check_policy_rest() {
log_bucket_policy() {
log 6 "log_bucket_policy"
if ! check_param_count "log_bucket_policy" "bucket" 1 $#; then
return
if ! check_param_count_gt "bucket, params (optional)" 1 $#; then
return 1
fi
if ! get_bucket_policy "rest" "$1"; then
log 2 "error getting bucket policy"
local bucket="$1"
local response policy
if ! response=$(get_bucket_policy_rest_go "$bucket" "${@:2}" 2>&1); then
if [[ "$response" == *"HTTP/1.1 404 Not Found"* ]]; then
log 5 "BUCKET POLICY: NONE"
return 0
fi
log 2 "error getting bucket policy: $response"
return
fi
policy="$response"
# shellcheck disable=SC2154
log 5 "BUCKET POLICY: $bucket_policy"
log 5 "BUCKET POLICY: '$policy'"
return 0
}
tests/util/util_versioning.sh
+35 -53
View File
@@ -100,34 +100,6 @@ echo_versions() {
return 0
}
get_base64_version_keys_and_ids() {
if ! check_param_count_v2 "'Version' or 'DeleteMarker'" 1 $#; then
return 1
fi
while IFS= read -r key && IFS= read -r vid; do
log 5 "key: $key, vid: $vid"
b_key="$(printf '%s' "$key" | base64 -w0)"
b_vid="$(printf '%s' "$vid" | base64 -w0)"
base64_pairs+=("$b_key:$b_vid")
done < <(xmlstarlet sel -t \
-m '//*[local-name()='"\"$1\""']' \
-v '*[local-name()="Key"]' -n \
-v '*[local-name()="VersionId"]' -n \
<<<"$versions" | xmlstarlet unesc)
}
parse_base64_versions_rest() {
base64_pairs=()
if ! get_base64_version_keys_and_ids "Version"; then
log 2 "error getting version base64 keys and IDs"
return 1
fi
if ! get_base64_version_keys_and_ids "DeleteMarker"; then
log 2 "error getting version base64 keys and IDs"
return 1
fi
}
parse_versions_rest() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
@@ -188,37 +160,44 @@ parse_versions_rest() {
}
get_and_check_versions_rest() {
if ! check_param_count_gt "bucket, key, count, expected islatest, expected id equal to null" 5 $#; then
if ! check_param_count_gt "bucket, key, count, expected islatest/expected id equal to null pairs" 5 $#; then
return 1
fi
if ! response=$(list_object_versions_rest "$1" 2>&1); then
local bucket="$1" key="$2" expected_count="$3" expected_pairs=("${@:4}")
local response versions_data versions_count id_check expected_islatest expected_id_is_null
if ! response=$(list_object_versions_rest "$bucket" 2>&1); then
log 2 "error listing object versions: $response"
return 1
fi
versions_file="$response"
versions_data="$response"
log 5 "versions: $(cat "$versions_file")"
if ! version_count=$(xmllint --xpath 'count(//*[local-name()="Version"])' "$versions_file" 2>&1); then
log 2 "error getting version count: $version_count"
log 5 "versions data: $versions_data"
if ! response=$(xmllint --xpath 'count(//*[local-name()="Version"])' - <<< "$versions_data" 2>&1); then
log 2 "error getting version count: $response"
return 1
fi
log 5 "version count: $version_count"
if [ "$version_count" != "$3" ]; then
log 2 "version count mismatch (expected 1, actual $version_count)"
versions_count="$response"
log 5 "version count: $versions_count"
if [ "$versions_count" != "$expected_count" ]; then
log 2 "version count mismatch (expected '1', actual '$versions_count')"
return 1
fi
while [ $# -ge 5 ]; do
if [ "$5" == "true" ]; then
for ((i=0; i<"${#expected_pairs[@]}"; i+=2)); do
expected_islatest="${expected_pairs[$i]}"
expected_id_is_null="${expected_pairs[$((i+1))]}"
if [ "$expected_id_is_null" == "true" ]; then
id_check="="
else
id_check="!="
fi
match_string="//*[local-name()=\"Version\"][*[local-name()=\"VersionId\" and text()$id_check\"null\"] and *[local-name()=\"IsLatest\" and text()=\"$4\"]]"
match_string="//*[local-name()=\"Version\"][*[local-name()=\"VersionId\" and text()${id_check}\"null\"] and *[local-name()=\"IsLatest\" and text()=\"${expected_islatest}\"]]"
log 5 "match string: $match_string"
if ! xmllint --xpath "$match_string" "$versions_file" 2>&1; then
if ! response=$(xmllint --xpath "$match_string" - <<< "$versions_data" 2>&1); then
log 2 "error matching: $response"
return 1
fi
shift 2
done
return 0
}
@@ -227,22 +206,25 @@ check_versions_after_file_deletion() {
if ! check_param_count "check_versions_after_file_deletion" "bucket, key" 2 $#; then
return 1
fi
if ! response=$(list_object_versions_rest "$1" 2>&1); then
local bucket="$1" key="$2"
local response versions_data version_key version_id marker_key marker_id
if ! response=$(list_object_versions_rest "$bucket" 2>&1); then
log 2 "error listing object versions: $response"
return 1
fi
versions_file="$response"
versions_data="$response"
log 5 "versions: $(cat "$versions_file")"
if ! version_key=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="Key"]/text()' "$versions_file" 2>&1); then
log 5 "versions: $(cat "$versions_data")"
if ! version_key=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="Key"]/text()' - <<< "$versions_data" 2>&1); then
log 2 "error getting Version 'Key' value: $version_key"
return 1
fi
if [ "$version_key" != "$2" ]; then
log 2 "version key mismatch (expected $2, actual $version_key)"
if [ "$version_key" != "$key" ]; then
log 2 "version key mismatch (expected '$key', actual '$version_key')"
return 1
fi
if ! version_id=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="VersionId"]/text()' "$versions_file" 2>&1); then
if ! version_id=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="VersionId"]/text()' - <<< "$versions_data" 2>&1); then
log 2 "error getting Version 'VersionID' value: $version_id"
return 1
fi
@@ -250,15 +232,15 @@ check_versions_after_file_deletion() {
log 2 "version ID mismatch (expected 'null', actual '$version_id')"
return 1
fi
if ! marker_key=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="Key"]/text()' "$versions_file" 2>&1); then
if ! marker_key=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="Key"]/text()' - <<< "$versions_data" 2>&1); then
log 2 "error getting Version 'Key' value: $marker_key"
return 1
fi
if [ "$marker_key" != "$2" ]; then
log 2 "delete marker key mismatch (expected $2, actual $marker_key)"
if [ "$marker_key" != "$key" ]; then
log 2 "delete marker key mismatch (expected '$key', actual '$marker_key')"
return 1
fi
if ! marker_id=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="VersionId"]/text()' "$versions_file" 2>&1); then
if ! marker_id=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="VersionId"]/text()' - <<< "$versions_data" 2>&1); then
log 2 "error getting Version 'VersionID' value: $versioning_info"
return 1
fi