diff --git a/tests/commands/get_bucket_location.sh b/tests/commands/get_bucket_location.sh index a0b030b5..631d7837 100644 --- a/tests/commands/get_bucket_location.sh +++ b/tests/commands/get_bucket_location.sh @@ -86,9 +86,13 @@ get_bucket_location_rest() { if ! check_param_count_v2 "bucket, callback" 2 $#; then return 1 fi - if ! send_rest_go_command_callback "200" "$2" "-bucketName" "$1" "-method" "GET" "-query" "location=" "-awsRegion" "$AWS_REGION"; then - log 2 "error sending rest go command" + local bucket="$1" callback="$2" + local response + + if ! response=$(send_rest_go_command_callback "200" "$callback" "-bucketName" "$bucket" "-method" "GET" "-query" "location=" 2>&1); then + log 2 "error sending rest go command: $response" return 1 fi + echo "$response" return 0 } \ No newline at end of file diff --git a/tests/commands/get_bucket_policy.sh b/tests/commands/get_bucket_policy.sh index cadeca90..f2bae0dc 100644 --- a/tests/commands/get_bucket_policy.sh +++ b/tests/commands/get_bucket_policy.sh @@ -19,23 +19,26 @@ get_bucket_policy() { if ! check_param_count "get_bucket_policy" "command type, bucket" 2 $#; then return 1 fi - local get_bucket_policy_result=0 - if [[ $1 == 's3api' ]]; then - get_bucket_policy_s3api "$2" || get_bucket_policy_result=$? - elif [[ $1 == 's3cmd' ]]; then - get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$? - elif [[ $1 == 'mc' ]]; then - get_bucket_policy_mc "$2" || get_bucket_policy_result=$? - elif [ "$1" == 'rest' ]; then - get_bucket_policy_rest "$2" || get_bucket_policy_result=$? + local command_type="$1" bucket="$2" + local response get_bucket_policy_result=0 + + if [[ $command_type == 's3api' ]]; then + response=$(get_bucket_policy_s3api "$bucket" 2>&1) || get_bucket_policy_result=$? + elif [[ $command_type == 's3cmd' ]]; then + response=$(get_bucket_policy_s3cmd "$bucket" 2>&1) || get_bucket_policy_result=$? + elif [[ $command_type == 'mc' ]]; then + response=$(get_bucket_policy_mc "$bucket" 2>&1) || get_bucket_policy_result=$? + elif [ "$command_type" == 'rest' ]; then + response=$(get_bucket_policy_rest "$bucket" 2>&1) || get_bucket_policy_result=$? else - log 2 "command 'get bucket policy' not implemented for '$1'" + log 2 "command 'get bucket policy' not implemented for '$command_type'" return 1 fi if [[ $get_bucket_policy_result -ne 0 ]]; then - log 2 "error getting policy: $bucket_policy" + log 2 "error getting policy: $response" return 1 fi + echo "$response" return 0 } @@ -44,19 +47,20 @@ get_bucket_policy_s3api() { if ! check_param_count "get_bucket_policy_s3api" "bucket" 1 $#; then return 1 fi - policy_json=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$? - policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning") - log 5 "$policy_json" - if [[ $get_result -ne 0 ]]; then - if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then + local bucket="$1" + local response policy_json bucket_policy + + if ! response=$(send_command aws --no-verify-ssl s3api get-bucket-policy --bucket "$bucket" 2>&1); then + if [[ "$response" == *"(NoSuchBucketPolicy)"* ]]; then bucket_policy= else - log 2 "error getting policy: $policy_json" + log 2 "error getting policy: $response" return 1 fi - else - bucket_policy=$(echo "$policy_json" | jq -r '.Policy') fi + policy_json=$(echo "$response" | grep -v "InsecureRequestWarning") + bucket_policy=$(echo "$policy_json" | jq -r '.Policy') + echo "$bucket_policy" return 0 } @@ -82,81 +86,138 @@ get_bucket_policy_s3cmd() { if ! check_param_count "get_bucket_policy_s3cmd" "bucket" 1 $#; then return 1 fi + local bucket="$1" + local response s3cmd_response bucket_policy - if ! info=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --region "$AWS_REGION" info "s3://$1" 2>&1); then - log 2 "error getting bucket policy: $info" + if ! response=$(send_command s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate --region "$AWS_REGION" info "s3://$bucket" 2>&1); then + log 2 "error getting bucket policy: $response" return 1 fi + s3cmd_response="$response" - log 5 "policy info: $info" - bucket_policy="" - policy_brackets=false + log 5 "policy info: $response" # NOTE: versitygw sends policies back in multiple lines here, direct in single line - while IFS= read -r line; do - if check_and_load_policy_info; then - break - fi - done <<< "$info" - log 5 "bucket policy: $bucket_policy" + if ! response=$(parse_s3cmd_policy "$s3cmd_response"); then + log 2 "error parsing s3cmd policy: $response" + return 1 + fi + log 5 "bucket policy: $response" + echo "$response" return 0 } +parse_s3cmd_policy() { + if ! check_param_count_v2 "response data" 1 $#; then + return 1 + fi + + local response_data="$1" + local line policy="" + + while IFS= read -r line; do + if [[ -z $policy ]]; then + [[ $line =~ ^[[:space:]]*Policy:[[:space:]]*(.*)$ ]] || continue + policy=${BASH_REMATCH[1]} + else + policy+=$'\n'"$line" + fi + + if [ "$policy" == "none" ]; then + echo "" + return 0 + elif jq -e . >/dev/null 2>&1 <<<"$policy"; then + printf '%s\n' "$policy" + return 0 + fi + done <<< "$response_data" + + log 2 "policy data not found (data: '$response_data')" + return 1 +} + get_bucket_policy_rest() { - if ! check_param_count "get_bucket_policy_rest" "bucket" 1 $#; then + if ! check_param_count_ge_le "bucket, region (optional)" 1 2 $#; then return 1 fi - log 5 "aws region: $AWS_REGION" - if ! get_bucket_policy_rest_expect_code "$1" "200"; then - log 2 "error getting REST bucket policy" + local bucket="$1" region="$2" + local response + + log 5 "aws region: $2" + if ! response=$(get_bucket_policy_rest_expect_code "$bucket" "200" "$region" 2>&1); then + log 2 "error getting REST bucket policy: $response" return 1 fi + echo "$response" + return 0 +} + +get_bucket_policy_rest_go() { + if ! check_param_count_gt "bucket, params (optional)" 1 $#; then + return 1 + fi + local bucket="$1" params=("${@:2}") + local response + + if ! response=$(send_rest_go_command "200" "-bucketName" "$bucket" "-query" "policy" "${params[@]}" 2>&1); then + log 2 "error getting bucket policy: $response" + return 1 + fi + echo "$response" return 0 } get_bucket_policy_rest_expect_code() { - if ! check_param_count "get_bucket_policy_rest_expect_code" "bucket, code" 2 $#; then + if ! check_param_count_ge_le "bucket, code, region (optional)" 2 3 $#; then return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/policy.txt" ./tests/rest_scripts/get_bucket_policy.sh); then - log 2 "error attempting to get bucket policy response: $result" - return 1 - fi - if [ "$result" != "$2" ]; then - log 2 "unexpected response code, expected '$2', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/policy.txt"))" - return 1 - fi - bucket_policy="$(cat "$TEST_FILE_FOLDER/policy.txt")" -} + local bucket_name="$1" expected_response_code="$2" region="$3" + local region_string response file_name -# return 0 for no policy, single-line policy, or loading complete, 1 for still searching or loading -check_and_load_policy_info() { - if [[ $policy_brackets == false ]]; then - if search_for_first_policy_line_or_full_policy; then - return 0 - fi + if [ "$region" != "" ]; then + region_string="AWS_REGION=$3" else - bucket_policy+=$line - if [[ $line == "}" ]]; then - return 0 - fi + region_string="AWS_REGION=$AWS_REGION" fi - return 1 + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! response=$(env COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket_name" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" "$region_string" ./tests/rest_scripts/get_bucket_policy.sh 2>&1); then + log 2 "error attempting to get bucket policy response: $response" + return 1 + fi + if [ "$response" != "$expected_response_code" ]; then + log 2 "unexpected response code, expected '$expected_response_code', actual '$response' (reply: '$(cat "$TEST_FILE_FOLDER/$file_name")')" + return 1 + fi + bucket_policy="$(cat "$TEST_FILE_FOLDER/$file_name")" + echo "$bucket_policy" + return 0 } -# return 0 for empty or single-line policy, 1 for other cases +# return 0 for empty or single-line policy, 1 for not found or in progress, 2 for error search_for_first_policy_line_or_full_policy() { - policy_line=$(echo "$line" | grep 'Policy: ') - if [[ $policy_line != "" ]]; then - if [[ $policy_line != *'{'* ]]; then + if ! check_param_count_v2 "line" 1 $#; then + return 2 + fi + local line="$1" + local policy_line + + if grep 'Policy: ' "$line" >/dev/null; then + if [[ $line != *'{'* ]]; then + echo "" return 0 fi - if [[ $policy_line == *'}'* ]]; then + if [[ $line == *'}'* ]]; then log 5 "policy on single line" - bucket_policy=${policy_line//Policy:/} + policy_line=${line//Policy:/} + echo "$policy_line" return 0 else - policy_brackets=true - bucket_policy+="{" + echo "{" fi fi return 1 diff --git a/tests/commands/get_object_lock_configuration.sh b/tests/commands/get_object_lock_configuration.sh index 81a6f7b1..7e2e178a 100644 --- a/tests/commands/get_object_lock_configuration.sh +++ b/tests/commands/get_object_lock_configuration.sh @@ -65,4 +65,19 @@ get_object_lock_configuration_rest() { echo "$response_code" "$TEST_FILE_FOLDER/$output_file" return $return_code +} + +get_object_lock_configuration_rest_go() { + if ! check_param_count_gt "bucket, callback, additional params" 2 $#; then + return 1 + fi + local bucket="$1" callback="$2" + local response + + if ! response=$(send_rest_go_command_callback "200" "$callback" "-query" "object-lock" "-bucketName" "$bucket" "${@:3}" 2>&1); then + log 2 "error getting object lock configuration: $response" + return 1 + fi + echo "$response" + return 0 } \ No newline at end of file diff --git a/tests/commands/list_object_versions.sh b/tests/commands/list_object_versions.sh index 7207ab31..5b70ee2b 100644 --- a/tests/commands/list_object_versions.sh +++ b/tests/commands/list_object_versions.sh @@ -18,17 +18,21 @@ list_object_versions() { if ! check_param_count "list_object_versions" "client, bucket name" 2 $#; then return 1 fi - local list_result=0 - if [ "$1" == "rest" ]; then - list_object_versions_rest "$2" || list_result=$? + local client="$1" bucket="$2" + local list_result=0 response response_with_warning versions + + if [ "$client" == "rest" ]; then + response=$(list_object_versions_rest "$bucket" 2>&1) || list_result=$? else - versions=$(send_command aws --no-verify-ssl s3api list-object-versions --bucket "$2" 2>&1) || list_result=$? + response_with_warning=$(send_command aws --no-verify-ssl s3api list-object-versions --bucket "$2" 2>&1) || list_result=$? + response=$(echo "$response_with_warning" | grep -v "InsecureRequestWarning") fi if [[ $list_result -ne 0 ]]; then log 2 "error listing object versions: $versions" return 1 fi - versions=$(echo "$versions" | grep -v "InsecureRequestWarning") + versions="$response" + echo "$versions" return 0 } @@ -51,18 +55,21 @@ list_object_versions_rest() { log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/$file_name"))" return 1 fi - versions=$(cat "$TEST_FILE_FOLDER/$file_name") - echo "$TEST_FILE_FOLDER/$file_name" + cat "$TEST_FILE_FOLDER/$file_name" return 0 } list_object_versions_rest_v2() { - if ! check_param_count_v2 "bucket name, callback" 2 $#; then + if ! check_param_count_gt "bucket name, callback, params (optional)" 2 $#; then return 1 fi - if ! send_rest_go_command_callback "200" "$2" "-bucketName" "$1" "-query" "versions="; then - log 2 "error sending REST list object versions command" + local bucket="$1" callback="$2" params=("${@:3}") + local response + + if ! response=$(send_rest_go_command_callback "200" "$callback" "-bucketName" "$bucket" "-query" "versions=" "${params[@]}" 2>&1); then + log 2 "error sending REST list object versions command: $response" return 1 fi + echo "$response" return 0 } \ No newline at end of file diff --git a/tests/commands/list_objects.sh b/tests/commands/list_objects.sh index 4743cbd4..2f9a7d3b 100644 --- a/tests/commands/list_objects.sh +++ b/tests/commands/list_objects.sh @@ -151,6 +151,21 @@ list_objects_rest() { return 0 } +list_objects_rest_go() { + if ! check_param_count_gt "bucket, callback, params (optional)" 2 $#; then + return 1 + fi + local bucket="$1" callback="$2" params=(${@:3}) + local response + + if ! response=$(send_rest_go_command_callback "200" "$callback" "-bucketName" "$bucket" "${params[@]}" 2>&1); then + log 2 "error sending ListObjects REST command: $response" + return 1 + fi + echo "$response" + return 0 +} + list_objects_rest_expect_error() { if ! check_param_count_v2 "bucket name, env vars, response code, error, message" 5 $#; then return 1 diff --git a/tests/drivers/delete_bucket/delete_bucket_rest.sh b/tests/drivers/delete_bucket/delete_bucket_rest.sh index ff455bdd..1d45e817 100644 --- a/tests/drivers/delete_bucket/delete_bucket_rest.sh +++ b/tests/drivers/delete_bucket/delete_bucket_rest.sh @@ -132,19 +132,21 @@ delete_bucket_recursive() { if ! check_param_count "delete_bucket_recursive_s3api" "bucket" 1 $#; then return 1 fi + local bucket="$1" + local response region endpoint - if ! location=$(get_bucket_location_rest "$1" "parse_bucket_location" 2>&1); then - log 2 "error getting bucket location: $location" + if ! response=$(get_bucket_location_and_endpoint "$bucket" 2>&1); then + log 2 "error getting bucket location and endpoint: $response" return 1 fi - log 5 "location: $location" + read -r region endpoint <<< "$response" - if ! reset_bucket "$1"; then + if ! reset_bucket "$bucket" "$region" "$endpoint"; then log 2 "error clearing bucket (s3api)" return 1 fi - if ! delete_bucket_rest "$1"; then + if ! AWS_REGION="$region" AWS_ENDPOINT_URL="$endpoint" delete_bucket_rest "$bucket"; then log 2 "error deleting bucket" return 1 fi diff --git a/tests/drivers/delete_object/delete_object_rest.sh b/tests/drivers/delete_object/delete_object_rest.sh index 2cf4c78e..080124af 100644 --- a/tests/drivers/delete_object/delete_object_rest.sh +++ b/tests/drivers/delete_object/delete_object_rest.sh @@ -20,12 +20,13 @@ source ./tests/drivers/xml.sh source ./tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh list_and_delete_objects() { - log 6 "list_and_delete_objects: '$1'" - if ! check_param_count_v2 "bucket" 1 $#; then + if ! check_param_count_v2 "bucket, lock config exists" 2 $#; then return 1 fi + local bucket="$1" lock_config_exists="$2" + local response - if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then + if ! response=$(list_objects_rest_go "$bucket" "parse_objects_list_rest" 2>&1); then log 2 "error getting object list: $response" return 1 fi @@ -35,44 +36,85 @@ list_and_delete_objects() { if [ "$object" == "" ]; then break fi - if ! clear_object_in_bucket "$1" "$object"; then + if ! clear_object_in_bucket "$bucket" "$object"; then log 2 "error deleting object $object" return 1 fi done - if ! delete_old_versions_base64 "$1"; then + if ! delete_old_versions "$bucket" "$lock_config_exists"; then log 2 "error deleting old version" return 1 fi return 0 } -delete_old_versions_base64() { - if ! check_param_count "delete_old_versions" "bucket" 1 $#; then +delete_old_versions() { + if ! check_param_count_v2 "bucket, lock config exists (true or false)" 2 $#; then return 1 fi - if ! list_object_versions "rest" "$1"; then - log 2 "error listing object versions" - return 1 - fi - # shellcheck disable=SC2154 - log 5 "versions: $versions" + local bucket="$1" lock_config_exists="$2" + local response versions_xml - if ! parse_base64_versions_rest; then - log 2 "error parsing version data" + if ! response=$(list_object_versions_rest_v2 "$bucket" "get_xml_versions_data" 2>&1); then + log 2 "error listing object versions: $response" return 1 fi + versions_xml="$response" - # shellcheck disable=SC2154 - log 5 "base64 versions: ${base64_pairs[*]}" - for pair in "${base64_pairs[@]}"; do - log 5 "pair: $pair" - if ! delete_object_version_with_or_without_retention_base64 "$1" "$pair"; then - log 2 "error deleting version with or without retention" + if ! response=$(get_and_delete_old_versions_from_xml "$versions_xml" "Version" "$bucket" "$lock_config_exists" 2>&1); then + log 2 "error deleting Versions: $response" + return 1 + fi + if ! response=$(get_and_delete_old_versions_from_xml "$versions_xml" "DeleteMarker" "$bucket" "$lock_config_exists" 2>&1); then + log 2 "error deleting DeleteMarkers: $response" + return 1 + fi + return 0 +} + +get_and_delete_old_versions_from_xml() { + if ! check_param_count_v2 "version XML, 'Version' or 'DeleteMarker', bucket, lock config exists" $# 4; then + return 1 + fi + local version_xml="$1" version_or_delete_marker="$2" bucket="$3" lock_config_exists="$4" + + while IFS= read -r key && IFS= read -r version_id; do + if ! delete_single_version_or_delete_marker "$bucket" "$lock_config_exists" "$key" "$version_id"; then + log 2 "error deleting key '$key', version ID '$version_id'" return 1 fi - done + done < <(xmlstarlet sel -t \ + -m '//*[local-name()='"\"$version_or_delete_marker\""']' \ + -v '*[local-name()="Key"]' -n \ + -v '*[local-name()="VersionId"]' -n \ + <<<"$version_xml" | xmlstarlet unesc) + return 0 +} + +delete_single_version_or_delete_marker() { + if ! check_param_count_v2 "bucket, lock config exists, key, version ID" 4 $#; then + return 1 + fi + local bucket="$1" lock_config="$2" key="$3" version_id="$4" + + if [ "$lock_config" == "true" ]; then + if ! check_remove_legal_hold_versions "$bucket" "$key" "$version_id"; then + log 2 "error checking, removing legal hold versions" + return 1 + fi + if ! delete_object_version_rest_bypass_retention "$bucket" "$key" "$version_id"; then + log 2 "error deleting object version, bypassing retention" + return 1 + fi + else + if ! delete_object_version_rest "$bucket" "$key" "$version_id"; then + log 2 "error deleting object version" + return 1 + fi + fi + log 5 "successfully deleted version with key '$key', id '$version_id'" + return 0 } delete_object_version_with_or_without_retention_base64() { diff --git a/tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh b/tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh index 2aedc838..0a788517 100644 --- a/tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh +++ b/tests/drivers/get_object_legal_hold/get_object_legal_hold_rest.sh @@ -30,12 +30,13 @@ check_legal_hold_without_lock_enabled() { } check_remove_legal_hold_versions() { - if ! check_param_count "check_remove_legal_hold_versions" "bucket, key, version ID" 3 $#; then + if ! check_param_count_gt "bucket, key, version ID" 3 $#; then return 1 fi + local bucket="$1" key="$2" version_id="$3" local response legal_hold_data - if ! response=$(get_object_legal_hold_rest_version_id "$1" "$2" "$3" 2>&1); then + if ! response=$(get_object_legal_hold_rest_version_id "$bucket" "$key" "$version_id" 2>&1); then # shellcheck disable=SC2154 log 5 "legal hold: $response" if [[ "$response" != *"MethodNotAllowed"* ]] && [[ "$response" != *"NoSuchObjectLockConfiguration"* ]]; then diff --git a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh index 3e4e4453..530cbef4 100644 --- a/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh +++ b/tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh @@ -63,3 +63,22 @@ check_object_lock_config_enabled_rest() { fi return 0 } + +check_object_lock_config_go() { + if ! check_param_count_gt "bucket, additional params (optional)" 1 $#; then + return 1 + fi + local bucket="$1" params=("${@:2}") + local response + + if ! response=$(get_object_lock_configuration_rest_go "$bucket" "" "${params[@]}" 2>&1); then + if [[ "$response" == *"HTTP/1.1 404 Not Found"* ]]; then + echo "false" + return 0 + fi + log 2 "error checking object lock configuration: $response" + return 1 + fi + echo "true" + return 0 +} diff --git a/tests/drivers/head_bucket/head_bucket_rest.sh b/tests/drivers/head_bucket/head_bucket_rest.sh index 3a97141d..a42cad95 100644 --- a/tests/drivers/head_bucket/head_bucket_rest.sh +++ b/tests/drivers/head_bucket/head_bucket_rest.sh @@ -38,3 +38,76 @@ check_bucket_existence_callback() { echo "error checking if bucket exists (data: $2)" return 2 } + +get_endpoint() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + local data_file="$1" + local response endpoint + + if ! response=$(get_element_text "$data_file" "Error" "Endpoint" 2>&1); then + log 2 "error getting endpoint: $response" + return 1 + fi + endpoint="$response" + log 5 "endpoint: $endpoint" + echo "$endpoint" + return 0 +} + +head_bucket_get_endpoint() { + if ! check_param_count_gt "bucket name, params (optional)" 1 $#; then + return 1 + fi + local bucket="$1" + local response endpoint + + if ! response=$(send_rest_go_command_expect_error_callback "301" "PermanentRedirect" "must be addressed" "get_endpoint" "-bucketName" "$bucket" "${@:2}" 2>&1); then + if [[ "$response" == *"HTTP/1.1 200 OK"* ]]; then + echo "$AWS_ENDPOINT_URL" + return 0 + fi + log 2 "error getting response: $response" + return 1 + fi + log 5 "response: $response" + endpoint="$response" + + log 5 "returned endpoint: $endpoint" + echo "https://${endpoint}" + return 0 +} + +get_bucket_location_and_endpoint() { + if ! check_param_count_v2 "bucket" 1 $#; then + return 1 + fi + local bucket="$1" + local response region endpoint_url + + if ! response=$(get_bucket_location_rest "$1" "parse_bucket_location" 2>&1); then + log 2 "error getting bucket location: $response" + return 1 + fi + region="$response" + if [ "$region" == "" ]; then + region="us-east-1" + fi + if [ "$DIRECT" != "true" ]; then + echo "$region $AWS_ENDPOINT_URL" + return 0 + fi + + if ! response=$(AWS_REGION="$region" head_bucket_get_endpoint "$bucket" 2>&1); then + log 2 "error getting bucket region and endpoint: $response" + return 1 + fi + if [ "$response" != "$AWS_ENDPOINT_URL" ]; then + endpoint_url="https://s3.${region}.amazonaws.com" + else + endpoint_url="$AWS_ENDPOINT_URL" + fi + echo "$region" "$endpoint_url" + return 0 +} diff --git a/tests/drivers/head_object/head_object_rest.sh b/tests/drivers/head_object/head_object_rest.sh index 5722e78a..26ff1229 100644 --- a/tests/drivers/head_object/head_object_rest.sh +++ b/tests/drivers/head_object/head_object_rest.sh @@ -256,32 +256,39 @@ check_header_partial_content_response() { } get_delete_marker_and_verify_405() { - if ! check_param_count "get_delete_marker_and_verify_405" "bucket, file name" 2 $#; then + if ! check_param_count_v2 "bucket, key" 2 $#; then return 1 fi - if ! response=$(list_object_versions_rest "$1" 2>&1); then + local bucket="$1" key="$2" + local response versions_data version_id file_name response_code + + if ! response=$(list_object_versions_rest "$bucket" 2>&1); then log 2 "error listing REST object versions" return 1 fi - versions_file="$response" - log 5 "versions: $(cat "$versions_file")" + versions_data="$response" + log 5 "versions: $versions_data" - if ! version_id=$(xmllint --xpath "//*[local-name()=\"DeleteMarker\"]/*[local-name()=\"VersionId\"]/text()" "$versions_file" 2>&1); then + if ! response=$(xmllint --xpath "//*[local-name()=\"DeleteMarker\"]/*[local-name()=\"VersionId\"]/text()" - <<< "$versions_data" 2>&1); then log 2 "error getting XML value: $version_id" return 1 fi - log 5 "xml val: $version_id" + version_id="$response" if ! response=$(get_file_name 2>&1); then log 2 "error getting file name: $response" return 1 fi - if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/$response" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then - log 2 "error getting result: $result" + file_name="$response" + + if ! response=$(OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then + log 2 "error getting result: $response" return 1 fi - if [ "$result" != "405" ]; then - log 2 "expected '405', was '$result' ($(cat "$TEST_FILE_FOLDER/$response"))" + response_code="$response" + + if [ "$response_code" != "405" ]; then + log 2 "expected '405', was '$response_code' ($(cat "$TEST_FILE_FOLDER/$file_name"))" return 1 fi return 0 diff --git a/tests/drivers/list_object_versions/list_object_versions_rest.sh b/tests/drivers/list_object_versions/list_object_versions_rest.sh index 9fc5b0e1..d56d5d77 100644 --- a/tests/drivers/list_object_versions/list_object_versions_rest.sh +++ b/tests/drivers/list_object_versions/list_object_versions_rest.sh @@ -272,4 +272,19 @@ list_object_versions_with_prefix_and_delimiter_check_results() { return 1 fi return 0 -} \ No newline at end of file +} + +get_xml_versions_data() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + local data_file="$1" + local response + + if ! response=$(get_element "$data_file" "ListVersionsResult" 2>&1); then + log 2 "error getting ListVersionsResult: $response" + return 1 + fi + printf '%s\n' "$response" + return 0 +} diff --git a/tests/drivers/list_objects/list_objects_rest.sh b/tests/drivers/list_objects/list_objects_rest.sh index 69796fde..0598cd65 100644 --- a/tests/drivers/list_objects/list_objects_rest.sh +++ b/tests/drivers/list_objects/list_objects_rest.sh @@ -356,17 +356,18 @@ parse_objects_list_rest() { if ! check_param_count_v2 "data file" 1 $#; then return 1 fi + local data_file="$1" + local response object_list object unescaped_object - local response - if ! response=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then - if [[ "$response" == *"XPath set is empty"* ]]; then + if ! response=$(get_element_text "$data_file" "Key" 2>&1); then + if [[ "$response" == *"element matching"*"doesn't exist"* ]]; then return 0 fi - log 2 "error getting object list: $response" + log 2 "error getting object keys: $response" return 1 - else - object_list="$response" fi + object_list="$response" + log 5 "object list: '$object_list'" while IFS= read -r object; do log 5 "parsed key: '$object'" diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index e1e572b4..0ddc194a 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -309,6 +309,7 @@ send_rest_go_command_expect_error_callback() { log 2 "callback error: $response" return 1 fi + echo "$response" return 0 } diff --git a/tests/drivers/user.sh b/tests/drivers/user.sh index aa9ed113..aeb222aa 100644 --- a/tests/drivers/user.sh +++ b/tests/drivers/user.sh @@ -119,61 +119,83 @@ setup_bucket_file_and_user_v2() { return 0 } -reset_bucket() { - if ! check_param_count "reset_bucket" "bucket" 1 $#; then +perform_reset() { + if ! check_param_count_v2 "bucket" 1 $#; then return 1 fi - log 6 "reset bucket '$1'" + local bucket="$1" local response lock_config_exists - if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$1"; then + if [[ $LOG_LEVEL_INT -ge 5 ]] && ! log_bucket_policy "$bucket"; then log 3 "error logging bucket policy" fi - if ! response=$(check_object_lock_config "$1" 2>&1); then + if ! response=$(check_object_lock_config_go "$bucket" 2>&1); then log 2 "error checking object lock config: $response" return 1 fi lock_config_exists="$response" - if [[ "$DIRECT" != "true" ]] && ! add_governance_bypass_policy "$1"; then + if [[ "$DIRECT" != "true" ]] && ! add_governance_bypass_policy "$bucket"; then log 2 "error adding governance bypass policy" return 1 fi - if ! list_and_delete_objects "$1"; then + if ! list_and_delete_objects "$bucket" "$lock_config_exists"; then log 2 "error listing and deleting objects" return 1 fi - if ! abort_all_multipart_uploads_rest "$1"; then + if ! abort_all_multipart_uploads_rest "$bucket"; then log 2 "error aborting all multipart uploads" return 1 fi - if [ "$SKIP_ACL_TESTING" != "true" ] && ! check_ownership_rule_and_reset_acl "$1"; then + if [ "$SKIP_ACL_TESTING" != "true" ] && ! check_ownership_rule_and_reset_acl "$bucket"; then log 2 "error checking ownership rule and resetting acl" return 1 fi - if ! delete_bucket_policy_rest "$1"; then + if ! delete_bucket_policy_rest "$bucket"; then log 2 "error deleting bucket policy" return 1 fi # shellcheck disable=SC2154 - if [[ $lock_config_exists == true ]] && ! remove_retention_policy_rest "$1"; then + if [[ $lock_config_exists == true ]] && ! remove_retention_policy_rest "$bucket"; then log 2 "error removing bucket retention policy" return 1 fi - if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$AWS_ACCESS_KEY_ID"; then + if [ "$RUN_USERS" == "true" ] && [ "$DIRECT" != "true" ] && ! change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$bucket" "$AWS_ACCESS_KEY_ID"; then log 2 "error changing bucket owner back to root" return 1 fi return 0 } +reset_bucket() { + if ! check_param_count_ge_le "bucket, region (optional), endpoint (optional)" 1 3 $#; then + return 1 + fi + local bucket="$1" region="$2" endpoint="$3" + local response + + if [ -z "$region" ] || [ -z "$endpoint" ]; then + if ! response=$(get_bucket_location_and_endpoint "$bucket" 2>&1); then + log 2 "error getting bucket region and endpoint: $response" + return 1 + fi + read -r region endpoint <<< "$response" + fi + + if ! response=$(AWS_REGION="$region" AWS_ENDPOINT_URL="$endpoint" perform_reset "$bucket" 2>&1); then + log 2 "error performing bucket reset: $response" + return 1 + fi + return 0 +} + get_user_id() { if [ "$DIRECT" == "true" ]; then if [ "$DIRECT_AWS_USER_ID" == "" ]; then @@ -185,4 +207,4 @@ get_user_id() { fi echo "$AWS_ACCESS_KEY_ID" return 0 -} \ No newline at end of file +} diff --git a/tests/test_rest_delete_bucket.sh b/tests/test_rest_delete_bucket.sh index c1873c00..40e02395 100755 --- a/tests/test_rest_delete_bucket.sh +++ b/tests/test_rest_delete_bucket.sh @@ -87,7 +87,7 @@ source ./tests/drivers/put_object/put_object_rest.sh run delete_bucket_rest_expect_error "$bucket_name" "" "409" "BucketNotEmpty" "is not empty" assert_success - run delete_old_versions_base64 "$bucket_name" + run delete_old_versions "$bucket_name" "false" assert_success run delete_bucket_rest "$bucket_name" diff --git a/tests/util/util_policy.sh b/tests/util/util_policy.sh index e1aa421c..62a50f1e 100644 --- a/tests/util/util_policy.sh +++ b/tests/util/util_policy.sh @@ -190,14 +190,18 @@ get_and_check_policy() { if ! check_param_count "get_and_check_policy" "client, bucket, expected effect, principal, action, resource" 6 $#; then return 1 fi - if ! get_bucket_policy "$1" "$2"; then - log 2 "error getting bucket policy after setting" + local client="$1" bucket="$2" expected_effect="$3" expected_principal="$4" expected_action="$5" expected_resource="$6" + local response policy + + if ! response=$(get_bucket_policy "$client" "$bucket" 2>&1); then + log 2 "error getting bucket policy after setting: $response" return 1 fi + policy="$response" # shellcheck disable=SC2154 - log 5 "POLICY: $bucket_policy" - if ! check_policy "$bucket_policy" "$3" "$4" "$5" "$6"; then + log 5 "POLICY: $policy" + if ! check_policy "$policy" "$expected_effect" "$expected_principal" "$expected_action" "$expected_resource"; then log 2 "error checking policy" return 1 fi @@ -304,28 +308,40 @@ get_and_compare_policy_with_file() { } put_and_check_policy_rest() { - if ! check_param_count "put_and_check_policy_rest" "bucket, policy file, effect, principal, action, resource" 6 $#; then + if ! check_param_count_v2 "bucket, policy file, effect, principal ID, action, resource" 6 $#; then return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" POLICY_FILE="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_bucket_policy.sh); then - log 2 "error putting policy: $result" + local bucket="$1" policy_file="$2" expected_effect="$3" principal_id="$4" expected_action="$5" expected_resource="$6" + local response file_name response_code policy expected_principal + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" return 1 fi - log 5 "response code: $result" - if [[ ( "$result" != "204" ) && ( "$result" != "200" ) ]]; then - log 2 "unexpected response code, expected '200' or '204', actual '$result' (reply: $(cat "$TEST_FILE_FOLDER/result.txt"))" + file_name="$response" + + if ! response=$(env COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" POLICY_FILE="$policy_file" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/put_bucket_policy.sh 2>&1); then + log 2 "error putting policy: $response" return 1 fi - if ! get_bucket_policy_rest "$1"; then - log 2 "error attempting to get bucket policy response: $result" + response_code="$response" + + if [[ ( "$response_code" != "204" ) && ( "$response_code" != "200" ) ]]; then + log 2 "unexpected response code, expected '200' or '204', actual '$response_code' (reply: '$(cat "$TEST_FILE_FOLDER/$file_name")')" return 1 fi + if ! response=$(get_bucket_policy_rest "$bucket" 2>&1); then + log 2 "error attempting to get bucket policy response: $response" + return 1 + fi + policy="$response" + if [ "$DIRECT" == "true" ]; then - principal="arn:aws:iam::$DIRECT_AWS_USER_ID:user/$4" + expected_principal="arn:aws:iam::$DIRECT_AWS_USER_ID:user/$principal_id" else - principal="$4" + expected_principal="$principal_id" fi - if ! check_policy "$bucket_policy" "$3" "$principal" "$5" "$6"; then + if ! check_policy "$policy" "$expected_effect" "$expected_principal" "$expected_action" "$expected_resource"; then log 2 "policies not equal" return 1 fi @@ -334,13 +350,23 @@ put_and_check_policy_rest() { log_bucket_policy() { log 6 "log_bucket_policy" - if ! check_param_count "log_bucket_policy" "bucket" 1 $#; then - return + if ! check_param_count_gt "bucket, params (optional)" 1 $#; then + return 1 fi - if ! get_bucket_policy "rest" "$1"; then - log 2 "error getting bucket policy" + local bucket="$1" + local response policy + + if ! response=$(get_bucket_policy_rest_go "$bucket" "${@:2}" 2>&1); then + if [[ "$response" == *"HTTP/1.1 404 Not Found"* ]]; then + log 5 "BUCKET POLICY: NONE" + return 0 + fi + log 2 "error getting bucket policy: $response" return fi + policy="$response" + # shellcheck disable=SC2154 - log 5 "BUCKET POLICY: $bucket_policy" + log 5 "BUCKET POLICY: '$policy'" + return 0 } diff --git a/tests/util/util_versioning.sh b/tests/util/util_versioning.sh index e4ae9c3a..4f385267 100644 --- a/tests/util/util_versioning.sh +++ b/tests/util/util_versioning.sh @@ -100,34 +100,6 @@ echo_versions() { return 0 } -get_base64_version_keys_and_ids() { - if ! check_param_count_v2 "'Version' or 'DeleteMarker'" 1 $#; then - return 1 - fi - while IFS= read -r key && IFS= read -r vid; do - log 5 "key: $key, vid: $vid" - b_key="$(printf '%s' "$key" | base64 -w0)" - b_vid="$(printf '%s' "$vid" | base64 -w0)" - base64_pairs+=("$b_key:$b_vid") - done < <(xmlstarlet sel -t \ - -m '//*[local-name()='"\"$1\""']' \ - -v '*[local-name()="Key"]' -n \ - -v '*[local-name()="VersionId"]' -n \ - <<<"$versions" | xmlstarlet unesc) -} - -parse_base64_versions_rest() { - base64_pairs=() - if ! get_base64_version_keys_and_ids "Version"; then - log 2 "error getting version base64 keys and IDs" - return 1 - fi - if ! get_base64_version_keys_and_ids "DeleteMarker"; then - log 2 "error getting version base64 keys and IDs" - return 1 - fi -} - parse_versions_rest() { if ! check_param_count_v2 "data file" 1 $#; then return 1 @@ -188,37 +160,44 @@ parse_versions_rest() { } get_and_check_versions_rest() { - if ! check_param_count_gt "bucket, key, count, expected islatest, expected id equal to null" 5 $#; then + if ! check_param_count_gt "bucket, key, count, expected islatest/expected id equal to null pairs" 5 $#; then return 1 fi - if ! response=$(list_object_versions_rest "$1" 2>&1); then + local bucket="$1" key="$2" expected_count="$3" expected_pairs=("${@:4}") + local response versions_data versions_count id_check expected_islatest expected_id_is_null + + if ! response=$(list_object_versions_rest "$bucket" 2>&1); then log 2 "error listing object versions: $response" return 1 fi - versions_file="$response" + versions_data="$response" - log 5 "versions: $(cat "$versions_file")" - if ! version_count=$(xmllint --xpath 'count(//*[local-name()="Version"])' "$versions_file" 2>&1); then - log 2 "error getting version count: $version_count" + log 5 "versions data: $versions_data" + if ! response=$(xmllint --xpath 'count(//*[local-name()="Version"])' - <<< "$versions_data" 2>&1); then + log 2 "error getting version count: $response" return 1 fi - log 5 "version count: $version_count" - if [ "$version_count" != "$3" ]; then - log 2 "version count mismatch (expected 1, actual $version_count)" + versions_count="$response" + + log 5 "version count: $versions_count" + if [ "$versions_count" != "$expected_count" ]; then + log 2 "version count mismatch (expected '1', actual '$versions_count')" return 1 fi - while [ $# -ge 5 ]; do - if [ "$5" == "true" ]; then + for ((i=0; i<"${#expected_pairs[@]}"; i+=2)); do + expected_islatest="${expected_pairs[$i]}" + expected_id_is_null="${expected_pairs[$((i+1))]}" + if [ "$expected_id_is_null" == "true" ]; then id_check="=" else id_check="!=" fi - match_string="//*[local-name()=\"Version\"][*[local-name()=\"VersionId\" and text()$id_check\"null\"] and *[local-name()=\"IsLatest\" and text()=\"$4\"]]" + match_string="//*[local-name()=\"Version\"][*[local-name()=\"VersionId\" and text()${id_check}\"null\"] and *[local-name()=\"IsLatest\" and text()=\"${expected_islatest}\"]]" log 5 "match string: $match_string" - if ! xmllint --xpath "$match_string" "$versions_file" 2>&1; then + if ! response=$(xmllint --xpath "$match_string" - <<< "$versions_data" 2>&1); then + log 2 "error matching: $response" return 1 fi - shift 2 done return 0 } @@ -227,22 +206,25 @@ check_versions_after_file_deletion() { if ! check_param_count "check_versions_after_file_deletion" "bucket, key" 2 $#; then return 1 fi - if ! response=$(list_object_versions_rest "$1" 2>&1); then + local bucket="$1" key="$2" + local response versions_data version_key version_id marker_key marker_id + + if ! response=$(list_object_versions_rest "$bucket" 2>&1); then log 2 "error listing object versions: $response" return 1 fi - versions_file="$response" + versions_data="$response" - log 5 "versions: $(cat "$versions_file")" - if ! version_key=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="Key"]/text()' "$versions_file" 2>&1); then + log 5 "versions: $(cat "$versions_data")" + if ! version_key=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="Key"]/text()' - <<< "$versions_data" 2>&1); then log 2 "error getting Version 'Key' value: $version_key" return 1 fi - if [ "$version_key" != "$2" ]; then - log 2 "version key mismatch (expected $2, actual $version_key)" + if [ "$version_key" != "$key" ]; then + log 2 "version key mismatch (expected '$key', actual '$version_key')" return 1 fi - if ! version_id=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="VersionId"]/text()' "$versions_file" 2>&1); then + if ! version_id=$(xmllint --xpath '//*[local-name()="Version"]/*[local-name()="VersionId"]/text()' - <<< "$versions_data" 2>&1); then log 2 "error getting Version 'VersionID' value: $version_id" return 1 fi @@ -250,15 +232,15 @@ check_versions_after_file_deletion() { log 2 "version ID mismatch (expected 'null', actual '$version_id')" return 1 fi - if ! marker_key=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="Key"]/text()' "$versions_file" 2>&1); then + if ! marker_key=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="Key"]/text()' - <<< "$versions_data" 2>&1); then log 2 "error getting Version 'Key' value: $marker_key" return 1 fi - if [ "$marker_key" != "$2" ]; then - log 2 "delete marker key mismatch (expected $2, actual $marker_key)" + if [ "$marker_key" != "$key" ]; then + log 2 "delete marker key mismatch (expected '$key', actual '$marker_key')" return 1 fi - if ! marker_id=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="VersionId"]/text()' "$versions_file" 2>&1); then + if ! marker_id=$(xmllint --xpath '//*[local-name()="DeleteMarker"]/*[local-name()="VersionId"]/text()' - <<< "$versions_data" 2>&1); then log 2 "error getting Version 'VersionID' value: $versioning_info" return 1 fi