diff --git a/tests/commands/list_objects.sh b/tests/commands/list_objects.sh index a4ff2480..bfeec7ed 100644 --- a/tests/commands/list_objects.sh +++ b/tests/commands/list_objects.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -source ./tests/util/util_list_objects.sh -source ./tests/commands/command.sh - # Copyright 2024 Versity Software # This file is licensed under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance @@ -17,6 +14,8 @@ source ./tests/commands/command.sh # specific language governing permissions and limitations # under the License. +source ./tests/commands/command.sh + # args: client, bucket name # return 0 if able to list, 1 if not list_objects() { @@ -48,15 +47,13 @@ list_objects() { return 1 fi - object_array=() while IFS= read -r line; do if [[ $line != *InsecureRequestWarning* ]]; then - object_name=$(echo "$line" | awk '{print $NF}') - object_array+=("$object_name") + printf '%s\n' "$line" fi done <<< "$output" - export object_array + return 0 } # args: bucket name @@ -74,13 +71,12 @@ list_objects_s3api() { log 5 "list_objects_s3api: raw data returned: $output" modified_output=$(echo "$output" | grep -v "InsecureRequestWarning") - object_array=() log 5 "modified output: $modified_output" - if echo "$modified_output" | jq -e 'has("Contents")'; then + if echo "$modified_output" | jq -e 'has("Contents")' >/dev/null; then contents=$(echo "$modified_output" | jq -r '.Contents[]') log 5 "contents: $contents" keys=$(echo "$contents" | jq -r '.Key') - IFS=$'\n' read -rd '' -a object_array <<<"$keys" + printf '%s\n' "$keys" fi return 0 } @@ -89,20 +85,25 @@ list_objects_s3api() { # param: bucket # export objects on success, return 1 for failure list_objects_s3api_v1() { - if [ $# -lt 1 ] || [ $# -gt 2 ]; then - log 2 "list objects command requires bucket, (optional) delimiter" + if ! check_param_count_ge_le "bucket, delimiter (optional)" 1 2 $#; then return 1 fi + local result=0 if [ "$2" == "" ]; then - objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$? + objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1) || result=$? else - objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$? + objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2" 2>&1) || result=$? fi if [[ $result -ne 0 ]]; then log 2 "error listing objects: $objects" return 1 fi - export objects + while IFS= read -r line; do + if [[ $line != *InsecureRequestWarning* ]]; then + printf '%s\n' "$line" + fi + done <<< "$objects" + return 0 } list_objects_with_prefix() { @@ -126,8 +127,11 @@ list_objects_with_prefix() { log 2 "error listing objects: $objects" return 1 fi - log 5 "output: $objects" - export objects + while IFS= read -r line; do + if [[ $line != *InsecureRequestWarning* ]]; then + printf '%s\n' "$line" + fi + done <<< "$objects" return 0 } diff --git a/tests/commands/list_objects_v2.sh b/tests/commands/list_objects_v2.sh index 4520a38b..a57bcabc 100644 --- a/tests/commands/list_objects_v2.sh +++ b/tests/commands/list_objects_v2.sh @@ -22,15 +22,22 @@ list_objects_v2() { log 2 "list objects command missing bucket and/or path" return 1 fi - objects=$(send_command aws --no-verify-ssl s3api list-objects-v2 --bucket "$1") || local result=$? + local result=0 + objects=$(send_command aws --no-verify-ssl s3api list-objects-v2 --bucket "$1" 2>&1) || result=$? if [[ $result -ne 0 ]]; then log 2 "error listing objects: $objects" return 1 fi + while IFS= read -r line; do + if [[ $line != *InsecureRequestWarning* ]]; then + printf '%s\n' "$line" + fi + done <<< "$objects" + return 0 } list_objects_v2_rest_callback() { - if ! check_param_count_gt "bucket, expected response code, callback fn, params" 3 $#; then + if ! check_param_count_gt "bucket, expected response code, callback fn, params, --, callback params" 3 $#; then return 1 fi if ! send_rest_go_command_callback "$2" "$3" "-bucketName" "$1" "-method" "GET" "-query" "list-type=2" "${@:4}"; then diff --git a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh index fcc2407d..7526db8e 100644 --- a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh +++ b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh @@ -15,16 +15,17 @@ # under the License. calculate_multipart_checksum() { - if ! check_param_count_gt "checksum type, part count, data file, checksums" 4 $#; then + if ! check_param_count_gt "checksum type, part count, data file, lowercase algorithm, checksums" 4 $#; then return 1 fi - log 5 "checksums: ${*:4}" + log 5 "checksums: ${*:5}" if [ "$1" == "COMPOSITE" ]; then - if ! calculate_composite_checksum "$lowercase_checksum_algorithm" "${@:4}"; then - log 2 "error calculating checksum" + if ! response=$(calculate_composite_checksum "$4" "${@:5}" 2>&1); then + log 2 "error calculating checksum: $response" return 1 fi - checksum="$composite-$2" + checksum="$response-$2" + echo "$checksum" return 0 fi @@ -32,42 +33,39 @@ calculate_multipart_checksum() { log 2 "unrecognized checksum type: $1" return 1 fi - if ! checksum=$(DATA_FILE="$3" CHECKSUM_TYPE="$lowercase_checksum_algorithm" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then - log 2 "error calculating checksum: $checksum" + if ! response=$(DATA_FILE="$3" CHECKSUM_TYPE="$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then + log 2 "error calculating checksum: $response" return 1 fi + echo "$response" return 0 } complete_multipart_upload_with_checksum() { - if ! check_param_count_v2 "bucket, key, file, upload ID, part count, checksum type, checksum algorithm" 7 $#; then + if ! check_param_count_v2 "bucket, key, file, upload ID, part count, parts payload, checksum type, checksum algorithm" 8 $#; then return 1 fi - log 5 "checksum algorithm: $7" - lowercase_checksum_algorithm=$(echo -n "$7" | tr '[:upper:]' '[:lower:]') - if ! upload_parts_rest_with_checksum_before_completion "$1" "$2" "$3" "$4" "$5" "$lowercase_checksum_algorithm"; then - log 2 "error uploading REST parts with checksum before completion" + log 5 "checksum algorithm: $8" + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" return 1 fi - log 5 "parts payload: $parts_payload" - log 5 "checksums: ${checksums[*]}" - if ! calculate_multipart_checksum "$6" "$5" "$3" "${checksums[@]}"; then - log 2 "error calculating multipart checksum" - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$4" PARTS="$parts_payload" CHECKSUM_TYPE="$6" CHECKSUM_ALGORITHM="$7" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then + response_file="$response" + + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$4" PARTS="$6" CHECKSUM_TYPE="$7" CHECKSUM_ALGORITHM="$8" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/$response_file" ./tests/rest_scripts/complete_multipart_upload.sh); then log 2 "error completing multipart upload" return 1 fi if [ "$result" != "200" ]; then - log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" + log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/$response_file"))" return 1 fi - log 5 "result: $(cat "$TEST_FILE_FOLDER/result.txt")" + log 5 "result: $(cat "$TEST_FILE_FOLDER/$response_file")" return 0 } calculate_composite_checksum() { + log 5 "algorithm: $1, checksums: ${*:2}" if ! check_param_count_gt "algorithm, at least two checksums" 3 $#; then return 1 fi @@ -96,6 +94,7 @@ calculate_composite_checksum() { fi log 5 "composite: $composite" echo "$composite" + return 0 } test_multipart_upload_with_checksum() { @@ -103,21 +102,29 @@ test_multipart_upload_with_checksum() { if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then return 1 fi - if ! file_and_bucket=$(setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" 2>&1); then + if ! file_and_bucket=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then log 2 "error setting up file and bucket" return 1 fi read -r bucket_name mp_file_name <<< "$file_and_bucket" log 5 "file name: $mp_file_name, file info: $(ls -l "$TEST_FILE_FOLDER/$mp_file_name")" - if ! perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2"; then - log 2 "error performing multipart upload with checksum before completion" + + if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then + log 2 "error performing multipart upload with checksum before completion: $response" return 1 fi - if ! calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "${checksums[@]}"; then + mapfile -t response_lines<<< "$response" + local lowercase_checksum_algorithm="${response_lines[0]}" + local upload_id="${response_lines[1]}" + local checksum_string="${response_lines[2]}" + local payload_parts="${response_lines[3]}" + read -r -a checksums <<< "$checksum_string" + + if ! calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}"; then log 2 "error calculating multipart checksum" return 1 fi - if ! complete_multipart_upload_with_checksum "$bucket_name" "$mp_file_name" "$TEST_FILE_FOLDER/$mp_file_name" "$upload_id" 2 "$1" "$2"; then + if ! complete_multipart_upload_with_checksum "$bucket_name" "$mp_file_name" "$TEST_FILE_FOLDER/$mp_file_name" "$upload_id" 2 "$payload_parts" "$1" "$2"; then log 2 "error completing multipart upload with checksum" return 1 fi @@ -128,16 +135,19 @@ test_complete_multipart_upload_unneeded_algorithm_parameter() { if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then return 1 fi - if ! file_and_bucket=$(setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" 2>&1); then + if ! file_and_bucket=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then log 2 "error setting up file and bucket" return 1 fi read -r bucket_name mp_file_name <<< "$file_and_bucket" - if ! perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2"; then + if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then log 2 "error performing multipart upload with checksum before completion" return 1 fi - log 5 "upload ID: $upload_id" + mapfile -t response_lines <<< "$response" + local upload_id="${response_lines[1]}" + local parts_payload="${response_lines[3]}" + if ! complete_multipart_upload_rest_nonexistent_param "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload"; then log 2 "error completing multipart upload with nonexistent param" return 1 @@ -149,21 +159,30 @@ test_complete_multipart_upload_incorrect_checksum() { if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then return 1 fi - if ! file_and_bucket=$(setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" 2>&1); then + if ! response=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then log 2 "error setting up file and bucket" return 1 fi - read -r bucket_name mp_file_name <<< "$file_and_bucket" - if ! perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2"; then + read -r bucket_name mp_file_name <<< "$response" + if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then log 2 "error performing multipart upload with checksum before completion" return 1 fi - if ! calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "${checksums[@]}"; then + mapfile -t response_lines<<< "$response" + local lowercase_checksum_algorithm="${response_lines[0]}" + local upload_id="${response_lines[1]}" + local checksum_string="${response_lines[2]}" + local payload_parts="${response_lines[3]}" + read -r -a checksums <<< "$checksum_string" + + if ! response=$(calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}" 2>&1); then log 2 "error calculating multipart checksum" return 1 fi - if ! complete_multipart_upload_rest_incorrect_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload" "$1" "$2" "$checksum"; then - log 2 "error completing multipart upload with nonexistent param" + checksum="$response" + + if ! complete_multipart_upload_rest_incorrect_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$payload_parts" "$1" "$2" "$checksum"; then + log 2 "error completing multipart upload with incorrect checksum" return 1 fi return 0 @@ -178,12 +197,16 @@ test_complete_multipart_upload_invalid_checksum() { return 1 fi read -r bucket_name mp_file_name <<< "$file_and_bucket" - if ! perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2"; then - log 2 "error performing multipart upload with checksum before completion" + if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then + log 2 "error performing multipart upload with checksum before completion: $response" return 1 fi + mapfile -t response_lines <<< "$response" + local upload_id="${response_lines[1]}" + local parts_payload="${response_lines[3]}" + if ! complete_multipart_upload_rest_invalid_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload" "$1" "$2" "wrong"; then - log 2 "error completing multipart upload with nonexistent param" + log 2 "error completing multipart upload with invalid checksum" return 1 fi return 0 @@ -270,3 +293,20 @@ perform_multipart_upload_rest_variable_parts() { fi return 0 } + +complete_multipart_upload_empty_upload_id() { + if ! check_param_count_v2 "bucket, key, file" 3 $#; then + return 1 + fi + local response upload_id parts_payload + if ! response=$(multipart_upload_rest_before_completion "$1" "$2" "$3" 2 2>&1); then + log 2 "error performing multipart upload before completion: $response" + return 1 + fi + read -r upload_id parts_payload <<< "$response" + if ! complete_multipart_upload_rest_expect_error "$1" "$2" "" "$parts_payload" "" "404" "NoSuchUpload" "The specified upload does not exist"; then + log 2 "error completing multipart upload" + return 1 + fi + return 0 +} diff --git a/tests/drivers/head_object/head_object_rest.sh b/tests/drivers/head_object/head_object_rest.sh index 566a57bb..369f1015 100644 --- a/tests/drivers/head_object/head_object_rest.sh +++ b/tests/drivers/head_object/head_object_rest.sh @@ -254,3 +254,35 @@ check_header_partial_content_response() { fi return 0 } + +get_delete_marker_and_verify_405() { + if ! check_param_count "get_delete_marker_and_verify_405" "bucket, file name" 2 $#; then + return 1 + fi + if ! response=$(list_object_versions_rest "$1" 2>&1); then + log 2 "error listing REST object versions" + return 1 + fi + versions_file="$response" + log 5 "versions: $(cat "$versions_file")" + + if ! version_id=$(xmllint --xpath "//*[local-name()=\"DeleteMarker\"]/*[local-name()=\"VersionId\"]/text()" "$versions_file" 2>&1); then + log 2 "error getting XML value: $version_id" + return 1 + fi + log 5 "xml val: $version_id" + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/$response" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then + log 2 "error getting result: $result" + return 1 + fi + if [ "$result" != "405" ]; then + log 2 "expected '405', was '$result' ($(cat "$TEST_FILE_FOLDER/$response"))" + return 1 + fi + return 0 +} diff --git a/tests/drivers/list_objects/list_objects.sh b/tests/drivers/list_objects/list_objects.sh new file mode 100644 index 00000000..e8a95a1a --- /dev/null +++ b/tests/drivers/list_objects/list_objects.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +list_check_objects_common() { + if ! check_param_count "list_check_objects_common" "client, bucket, object one, object two" 4 $#; then + return 1 + fi + if ! response=$(list_objects "$1" "$2" 2>&1); then + log 2 "error listing objects: $response" + return 1 + fi + log 5 "response: $response" + mapfile -t object_list <<< "$response" + + local object_one_found=false + local object_two_found=false + for object_data in "${object_list[@]}"; do + if [ "$1" == "s3cmd" ]; then + object="$(echo -n "$object_data" | awk '{ for (i=4; i<=NF; i++) printf "%s%s", $i, (i&1); then + log 2 "error listing objects: $response" + return 1 + fi + mapfile -t object_array <<< "$response" + + if [[ $LOG_LEVEL -ge 5 ]]; then + log 5 "Array: ${object_array[*]}" + fi + local file_count="${#object_array[@]}" + if [[ $file_count != "$3" ]]; then + log 2 "file count should be $3, is $file_count" + return 1 + fi + return 0 +} diff --git a/tests/drivers/list_objects/list_objects_rest.sh b/tests/drivers/list_objects/list_objects_rest.sh index 56ee1e71..b7ce43a0 100644 --- a/tests/drivers/list_objects/list_objects_rest.sh +++ b/tests/drivers/list_objects/list_objects_rest.sh @@ -56,14 +56,15 @@ list_objects_success_or_access_denied() { } check_v2_objects() { - if ! check_param_count_v2 "data file" 1 $#; then + if ! check_param_count_gt "data file, expected objects" 1 $#; then return 1 fi + object_count=$(($#-1)) if ! check_xml_element "$1" "$object_count" "ListBucketResult" "KeyCount"; then log 2 "error checking KeyCount element" return 1 fi - for object in "${expected_objects[@]}"; do + for object in "${@:2}"; do if ! check_if_element_exists "$1" "$object" "ListBucketResult" "Contents" "Key"; then log 2 "error checking if element '$object' exists" return 1 @@ -78,7 +79,7 @@ list_check_objects_rest_v2() { fi object_count=$2 expected_objects=("${@:3:$object_count}") - if ! list_objects_v2_rest_callback "$1" "200" "check_v2_objects" "${@:((3+$object_count))}"; then + if ! list_objects_v2_rest_callback "$1" "200" "check_v2_objects" "${@:((3+$object_count))}" "--" "${expected_objects[@]}"; then log 2 "error sending list objects v2 command and checking callback" return 1 fi @@ -420,3 +421,215 @@ list_objects_delimiter() { run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "2" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt" assert_success } + +parse_objects_list_rest() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + + local response + if ! response=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then + if [[ "$response" == *"XPath set is empty"* ]]; then + return 0 + fi + log 2 "error getting object list: $response" + return 1 + else + object_list="$response" + fi + log 5 "object list: '$object_list'" + while IFS= read -r object; do + log 5 "parsed key: '$object'" + unescaped_object="$(echo -n "$object" | xmlstarlet unesc)" + echo "$unescaped_object" + done <<< "$object_list" + return 0 +} + +list_check_objects_rest() { + if ! check_param_count "list_check_objects_rest" "bucket" 1 $#; then + return 1 + fi + + local response + if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then + log 2 "error listing and parsing objects: $response" + return 1 + fi + mapfile -t object_array <<< "$response" + + object_found=false + # shellcheck disable=SC2154 + for object in "${object_array[@]}"; do + log 5 "object: $object" + if [[ $object == "$test_file" ]]; then + object_found=true + break + fi + done + if [[ $object_found == "false" ]]; then + log 2 "object not found" + return 1 + fi + return 0 +} + +list_objects_with_user_rest_verify_access_denied() { + if ! check_param_count "list_objects_with_user_rest_verify_access_denied" "bucket, username, password" 3 $#; then + return 1 + fi + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then + log 2 "error attempting to list objects: $result" + return 1 + fi + if [ "$result" != "403" ]; then + log 2 "expected response code of '403', was '$result'" + return 1 + fi + error_message="$(cat "$TEST_FILE_FOLDER/$file_name")" + if [[ "$error_message" != *"Access Denied"* ]]; then + log 2 "unexpected error message: $error_message" + return 1 + fi + return 0 +} + +list_objects_with_user_rest_verify_success() { + if ! check_param_count "list_objects_with_user_rest_verify_success" "bucket, username, password, expected object" 4 $#; then + return 1 + fi + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then + log 2 "error attempting to list objects: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "expected response code of '200', was '$result' (error: $(cat "$TEST_FILE_FOLDER/$file_name"))" + return 1 + fi + if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/$file_name" 2>&1); then + log 2 "error getting object key: $key" + return 1 + fi + if [ "$key" != "$4" ]; then + log 2 "expected '$4', was '$key'" + return 1 + fi + return 0 +} + +list_objects_check_params_get_token() { + if ! check_param_count "list_objects_check_params_get_token" "bucket, files, version two" 4 $#; then + return 1 + fi + if ! response=$(get_file_names 2 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + read -r file_name list_bucket_file <<< "$response" + + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="$4" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then + log 2 "error attempting to get bucket ACL response: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "expected '200' was '$result' ($(cat "$TEST_FILE_FOLDER/$file_name"))" + return 1 + fi + log 5 "objects: $(cat "$TEST_FILE_FOLDER/$file_name")" + if ! list_bucket_result=$(xmllint --xpath '//*[local-name()="ListBucketResult"]' "$TEST_FILE_FOLDER/$file_name" 2>&1); then + log 2 "error getting list bucket result: $list_bucket_result" + return 1 + fi + echo -n "$list_bucket_result" > "$TEST_FILE_FOLDER/$list_bucket_file" + if ! check_xml_element "$TEST_FILE_FOLDER/$list_bucket_file" "$2" "Key"; then + log 2 "key mismatch" + return 1 + fi + if ! check_xml_element "$TEST_FILE_FOLDER/$list_bucket_file" "1" "MaxKeys"; then + log 2 "max keys mismatch" + return 1 + fi + if ! check_xml_element "$TEST_FILE_FOLDER/$list_bucket_file" "1" "KeyCount"; then + log 2 "key count mismatch" + return 1 + fi + if ! check_xml_element "$TEST_FILE_FOLDER/$list_bucket_file" "true" "IsTruncated"; then + log 2 "key count mismatch" + return 1 + fi + if ! continuation_token=$(xmllint --xpath '//*[local-name()="NextContinuationToken"]/text()' "$TEST_FILE_FOLDER/$list_bucket_file" 2>&1); then + log 2 "error getting next continuation token: $continuation_token" + return 1 + fi + echo "$continuation_token" + return 0 +} + +list_objects_check_continuation_error() { + if ! check_param_count "list_objects_check_continuation_error" "bucket, continuation token" 2 $#; then + return 1 + fi + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 CONTINUATION_TOKEN="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then + log 2 "error attempting to get bucket ACL response: $result" + return 1 + fi + if [ "$result" != "400" ]; then + log 2 "expected result code of '400' was '$result'" + return 1 + fi + if ! check_xml_element "$TEST_FILE_FOLDER/$file_name" "InvalidArgument" "Error" "Code"; then + log 2 "invalid error code" + return 1 + fi + return 0 +} + +list_objects_v1_check_nextmarker_empty() { + if ! check_param_count "list_objects_v1_check_nextmarker_empty" "bucket" 1 $#; then + return 1 + fi + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then + log 2 "error attempting to get bucket ACL response: $result" + return 1 + fi + log 5 "output: $(cat "$TEST_FILE_FOLDER/$file_name")" + if ! next_marker=$(xmllint --xpath '//*[local-name()="NextMarker"]' "$TEST_FILE_FOLDER/$file_name" 2>&1); then + if [[ "$next_marker" != *"XPath set is empty"* ]]; then + log 2 "unexpected error: $next_marker" + return 1 + fi + return 0 + fi + log 5 "next marker: $next_marker" + marker_text=$(xmllint --xpath 'string(/NextMarker)' <(echo "$next_marker") 2>&1) + log 5 "marker text: $marker_text" + if [[ "$marker_text" != *"Document is empty"* ]]; then + log 2 "NextMarker text should be empty, but is $marker_text" + return 1 + fi + return 0 +} diff --git a/tests/drivers/list_objects/list_objects_s3api.sh b/tests/drivers/list_objects/list_objects_s3api.sh new file mode 100644 index 00000000..44b0dce2 --- /dev/null +++ b/tests/drivers/list_objects/list_objects_s3api.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +list_check_objects_v1() { + if ! check_param_count_gt "bucket, expected keys" 1 $#; then + return 1 + fi + if ! response=$(list_objects_s3api_v1 "$1" 2>&1); then + log 2 "error listing objects (s3api, v1): $response" + return 1 + fi + object_json="$response" + log 5 "object JSON: $response" + + if ! check_listed_objects "$object_json" "${@:2}"; then + log 2 "error checking listed objects v2" + return 1 + fi + return 0 +} + +check_listed_objects() { + if ! check_param_count_gt "JSON data, expected keys (if any)" 1 $#; then + return 1 + fi + expected_key_count="$(($#-1))" + if ! response=$(echo "$1" | jq '.Contents | length' 2>&1); then + log 2 "error getting length: $response" + return 1 + fi + key_count="$response" + if [ "$key_count" -ne "$expected_key_count" ]; then + log 2 "expected key count of '$expected_key_count', was '$key_count'" + return 1 + fi + + declare -A key_array size_array + for ((i=0; i&1); then + log 2 "error obtaining key one: $response" + return 1 + fi + key="$response" + key_array["$key"]="true" + if ! response=$(echo "$1" | jq -r ".Contents[$i].Size" 2>&1); then + log 2 "error obtaining key one: $response" + return 1 + fi + size_array["$key"]="$response" + done + + for expected_key in "${@:2}"; do + if [ "${key_array[$expected_key]}" != "true" ]; then + continue + fi + if ! response=$(get_file_size "$TEST_FILE_FOLDER/$expected_key" 2>&1); then + log 2 "error getting file size: $response" + return 1 + fi + if [ "${size_array[$expected_key]}" != "$response" ]; then + log 2 "size mismatch, expected '${size_array[$expected_key]}', was $response" + return 1 + fi + unset key_array["$expected_key"] + done + if [ ${#key_array[@]} -ne 0 ]; then + log 2 "key mismatch: keys left over: '${!key_array[*]}'" + return 1 + fi + return 0 +} + +list_check_objects_v2() { + if ! check_param_count_gt "bucket, expected keys" 1 $#; then + return 1 + fi + if ! response=$(list_objects_v2 "$1" 2>&1); then + log 2 "error listing objects (s3api, v2): $response" + return 1 + fi + object_json="$response" + + if ! check_listed_objects "$object_json" "${@:2}"; then + log 2 "error checking listed objects" + return 1 + fi + return 0 +} + +check_object_listing_with_prefixes() { + if ! check_param_count "check_object_listing_with_prefixes" "bucket, folder name, object name" 3 $#; then + return 1 + fi + if ! response=$(list_objects_s3api_v1 "$1" "/" 2>&1); then + log 2 "error listing objects with delimiter '/': $response" + return 1 + fi + object_json="$response" + + if ! prefix=$(echo "$object_json" | jq -r ".CommonPrefixes[0].Prefix" 2>&1); then + log 2 "error getting object prefix from object list: $prefix" + return 1 + fi + if [[ $prefix != "$2/" ]]; then + log 2 "prefix doesn't match (expected $2, actual $prefix/)" + return 1 + fi + if ! response=$(list_objects_s3api_v1 "$1" "#" 2>&1); then + log 2 "error listing objects with delimiter '#: $response" + return 1 + fi + objects="$response" + + if ! key=$(echo "$objects" | jq -r ".Contents[0].Key" 2>&1); then + log 2 "error getting key from object list: $key" + return 1 + fi + if [[ $key != "$2/$3" ]]; then + log 2 "key doesn't match (expected $key, actual $2/$3)" + return 1 + fi + return 0 +} diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index f65d3f1c..271a5a73 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -14,6 +14,7 @@ # specific language governing permissions and limitations # under the License. +source ./tests/drivers/list_objects/list_objects_rest.sh source ./tests/drivers/xml.sh check_rest_expected_error() { diff --git a/tests/drivers/upload_part/upload_part_rest.sh b/tests/drivers/upload_part/upload_part_rest.sh index 521b4917..e7d05cb7 100644 --- a/tests/drivers/upload_part/upload_part_rest.sh +++ b/tests/drivers/upload_part/upload_part_rest.sh @@ -85,6 +85,8 @@ upload_parts_rest_with_checksum_before_completion() { log 5 "parts payload: $parts_payload" done log 5 "CHECKSUMS: ${checksums[*]}" + echo "${checksums[*]}" + echo "$parts_payload" return 0 } @@ -92,25 +94,21 @@ perform_full_multipart_upload_with_checksum_before_completion() { if ! check_param_count_v2 "bucket, filename, checksum type, algorithm" 4 $#; then return 1 fi - if ! bucket_name=$(get_bucket_name "$1" 2>&1); then - log 2 "error getting bucket name: $bucket_name" - return 1 - fi - if ! setup_bucket_and_large_file_v2 "$bucket_name" "$2"; then - log 2 "error setting up bucket and large file" - return 1 - fi + lowercase_checksum_algorithm=$(echo -n "$4" | tr '[:upper:]' '[:lower:]') local response - if ! response=$(create_multipart_upload_rest "$bucket_name" "$2" "CHECKSUM_TYPE=$3 CHECKSUM_ALGORITHM=$4" "parse_upload_id" 2>&1); then + if ! response=$(create_multipart_upload_rest "$1" "$2" "CHECKSUM_TYPE=$3 CHECKSUM_ALGORITHM=$4" "parse_upload_id" 2>&1); then log 2 "error creating multipart upload: $response" return 1 fi upload_id="$response" - lowercase_checksum_algorithm=$(echo -n "$4" | tr '[:upper:]' '[:lower:]') - if ! upload_parts_rest_with_checksum_before_completion "$bucket_name" "$2" "$TEST_FILE_FOLDER/$2" "$upload_id" 2 "$lowercase_checksum_algorithm"; then - log 2 "error uploading parts" + if ! response=$(upload_parts_rest_with_checksum_before_completion "$1" "$2" "$TEST_FILE_FOLDER/$2" "$upload_id" 2 "$lowercase_checksum_algorithm" 2>&1); then + log 2 "error uploading parts: $response" return 1 fi + echo "$lowercase_checksum_algorithm" + echo "$upload_id" + echo "$response" + return 0 } \ No newline at end of file diff --git a/tests/test_rest_list_objects.sh b/tests/test_rest_list_objects.sh index e4b9af7d..f891df25 100755 --- a/tests/test_rest_list_objects.sh +++ b/tests/test_rest_list_objects.sh @@ -19,7 +19,9 @@ load ./bats-assert/load source ./tests/setup.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/list_objects/list_objects.sh source ./tests/drivers/list_objects/list_objects_rest.sh +source ./tests/drivers/list_objects/list_objects_s3api.sh # tags: curl,ListObjects,minimal-request @test "test_rest_list_objects" { diff --git a/tests/test_rest_multipart.sh b/tests/test_rest_multipart.sh index 42d0db0c..67de6b5c 100755 --- a/tests/test_rest_multipart.sh +++ b/tests/test_rest_multipart.sh @@ -464,3 +464,15 @@ source ./tests/drivers/upload_part/upload_part_rest.sh run complete_multipart_upload_invalid_object_size_string "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file" assert_success } + +@test "REST - multipart - empty upload ID on completion" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/2146" + fi + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run complete_multipart_upload_empty_upload_id "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/$file_name" + assert_success +} diff --git a/tests/test_s3_file_count.sh b/tests/test_s3_file_count.sh index 92674498..e0fbc871 100755 --- a/tests/test_s3_file_count.sh +++ b/tests/test_s3_file_count.sh @@ -17,6 +17,7 @@ load ./bats-support/load load ./bats-assert/load +source ./tests/drivers/list_objects/list_objects.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/test_common.sh diff --git a/tests/test_s3api_object.sh b/tests/test_s3api_object.sh index d1ee336d..629ff6c6 100755 --- a/tests/test_s3api_object.sh +++ b/tests/test_s3api_object.sh @@ -48,6 +48,8 @@ source ./tests/drivers/get_object_legal_hold/get_object_legal_hold.sh source ./tests/drivers/get_object_retention/get_object_retention_s3api.sh source ./tests/drivers/get_object_tagging/get_object_tagging.sh source ./tests/drivers/list_buckets/list_buckets_rest.sh +source ./tests/drivers/list_objects/list_objects.sh +source ./tests/drivers/list_objects/list_objects_s3api.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/drivers/file.sh source ./tests/util/util_lock_config.sh diff --git a/tests/test_s3api_root_inner.sh b/tests/test_s3api_root_inner.sh index 3075c726..58190c09 100755 --- a/tests/test_s3api_root_inner.sh +++ b/tests/test_s3api_root_inner.sh @@ -18,86 +18,90 @@ source ./tests/commands/delete_objects.sh source ./tests/commands/list_objects_v2.sh source ./tests/commands/list_parts.sh source ./tests/drivers/put_object/put_object.sh -source ./tests/util/util_list_objects.sh test_delete_objects_s3api_root() { - local object_one="test-file-one" - local object_two="test-file-two" - run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two" + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name object_one object_two <<< "$output" + + run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$bucket_name" "$object_one" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$BUCKET_ONE_NAME" "$object_one" + run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$bucket_name" "$object_two" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$BUCKET_ONE_NAME" "$object_two" + run delete_objects "$bucket_name" "$object_one" "$object_two" assert_success - run delete_objects "$BUCKET_ONE_NAME" "$object_one" "$object_two" - assert_success - - run object_exists "s3api" "$BUCKET_ONE_NAME" "$object_one" + run object_exists "s3api" "$bucket_name" "$object_one" assert_failure 1 - run object_exists "s3api" "$BUCKET_ONE_NAME" "$object_two" + run object_exists "s3api" "$bucket_name" "$object_two" assert_failure 1 } test_get_object_full_range_s3api_root() { - bucket_file="bucket_file" + run get_file_name + assert_success + bucket_file="$output" echo -n "0123456789" > "$TEST_FILE_FOLDER/$bucket_file" - run setup_bucket "$BUCKET_ONE_NAME" + run setup_bucket_v3 "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" - assert_success - - run get_object_with_range "$BUCKET_ONE_NAME" "$bucket_file" "bytes=9-15" "$TEST_FILE_FOLDER/$bucket_file-range" + run get_object_with_range "$bucket_name" "$bucket_file" "bytes=9-15" "$TEST_FILE_FOLDER/$bucket_file-range" assert_success assert [ "$(cat "$TEST_FILE_FOLDER/$bucket_file-range")" == "9" ] } test_get_object_invalid_range_s3api_root() { - bucket_file="bucket_file" - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file" + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name bucket_file <<< "$output" + + run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" - assert_success - - run get_object_with_range "$BUCKET_ONE_NAME" "$bucket_file" "bytes=0-0" "$TEST_FILE_FOLDER/$bucket_file-range" + run get_object_with_range "$bucket_name" "$bucket_file" "bytes=0-0" "$TEST_FILE_FOLDER/$bucket_file-range" assert_success } test_put_object_s3api_root() { - bucket_file="bucket_file" + run get_file_name + assert_success + bucket_file="$output" run create_test_files "$bucket_file" assert_success - run setup_buckets "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" + run setup_buckets_v3 "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" + assert_success + read -r bucket_one bucket_two <<< "$output" + + run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$bucket_one" "$bucket_file" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" + run copy_object "s3api" "$bucket_one/$bucket_file" "$bucket_two" "$bucket_file" assert_success - run copy_object "s3api" "$BUCKET_ONE_NAME/$bucket_file" "$BUCKET_TWO_NAME" "$bucket_file" - assert_success - - run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/${bucket_file}_copy" + run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_two" "$bucket_file" "$TEST_FILE_FOLDER/${bucket_file}_copy" assert_success } test_get_object_attributes_s3api_root() { - bucket_file="bucket_file" - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file" + run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name bucket_file <<< "$output" + + run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" - assert_success - - run get_and_check_object_size "$BUCKET_ONE_NAME" "$bucket_file" 10 + run get_and_check_object_size "$bucket_name" "$bucket_file" 10 assert_success } @@ -233,33 +237,31 @@ test_retention_bypass_s3api_root() { } test_s3api_list_objects_v1_s3api_root() { - local object_one="test-file-one" - local object_two="test-file-two" - run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two" + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name object_one object_two <<< "$output" + + run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$bucket_name" "$object_one" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$BUCKET_ONE_NAME" "$object_one" + run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$bucket_name" "$object_two" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$BUCKET_ONE_NAME" "$object_two" - assert_success - - run list_check_objects_v1 "$BUCKET_ONE_NAME" "$object_one" 10 "$object_two" 10 + run list_check_objects_v1 "$bucket_name" "$object_one" "$object_two" assert_success } test_s3api_list_objects_v2_s3api_root() { - local object_one="test-file-one" - local object_two="test-file-two" - run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two" + run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 + assert_success + read -r bucket_name object_one object_two <<< "$output" + + run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$bucket_name" "$object_one" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$BUCKET_ONE_NAME" "$object_one" + run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$bucket_name" "$object_two" assert_success - run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$BUCKET_ONE_NAME" "$object_two" - assert_success - - run list_check_objects_v2 "$BUCKET_ONE_NAME" "$object_one" 10 "$object_two" 10 + run list_check_objects_v2 "$bucket_name" "$object_one" "$object_two" assert_success } diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index e8a9c070..d30be653 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -25,6 +25,7 @@ source ./tests/commands/get_bucket_policy.sh source ./tests/commands/put_bucket_policy.sh source ./tests/drivers/get_bucket_location/get_bucket_location.sh source ./tests/drivers/list_buckets/list_buckets.sh +source ./tests/drivers/list_objects/list_objects.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/util/util_object.sh diff --git a/tests/test_s3cmd_file_count.sh b/tests/test_s3cmd_file_count.sh index 2a32563e..a3393b2d 100755 --- a/tests/test_s3cmd_file_count.sh +++ b/tests/test_s3cmd_file_count.sh @@ -17,6 +17,7 @@ load ./bats-support/load load ./bats-assert/load +source ./tests/drivers/list_objects/list_objects.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh source ./tests/test_common.sh diff --git a/tests/util/util_list_objects.sh b/tests/util/util_list_objects.sh deleted file mode 100644 index f29417ae..00000000 --- a/tests/util/util_list_objects.sh +++ /dev/null @@ -1,376 +0,0 @@ -#!/usr/bin/env bash - -source ./tests/commands/list_objects_v2.sh -source ./tests/drivers/xml.sh - -# Copyright 2024 Versity Software -# This file is licensed under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -parse_objects_list_rest() { - if ! check_param_count_v2 "data file" 1 $#; then - return 1 - fi - - local response - if ! response=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then - if [[ "$response" == *"XPath set is empty"* ]]; then - return 0 - fi - log 2 "error getting object list: $response" - return 1 - else - object_list="$response" - fi - log 5 "object list: '$object_list'" - while IFS= read -r object; do - log 5 "parsed key: '$object'" - unescaped_object="$(echo -n "$object" | xmlstarlet unesc)" - echo "$unescaped_object" - done <<< "$object_list" - return 0 -} - -list_check_objects_v1() { - if ! check_param_count "list_check_objects_v1" "bucket, expected key one, expected size one, expected key two, expected size two" 5 $#; then - return 1 - fi - if ! list_objects_s3api_v1 "$1"; then - log 2 "error listing objects (s3api, v1)" - return 1 - fi - if ! check_listed_objects "$2" "$3" "$4" "$5"; then - log 2 "error checking listed objects" - return 1 - fi - return 0 -} - -check_listed_objects() { - if ! check_param_count "check_listed_objects" "expected key one, expected size one, expected key two, expected size two" 4 $#; then - return 1 - fi - # shellcheck disable=SC2154 - if ! key_one=$(echo "$objects" | jq -r '.Contents[0].Key' 2>&1); then - log 2 "error obtaining key one: $key_one" - return 1 - fi - if [[ $key_one != "$1" ]]; then - log 2 "Object one mismatch ($key_one, $1)" - return 1 - fi - if ! size_one=$(echo "$objects" | jq -r '.Contents[0].Size' 2>&1); then - log 2 "error obtaining size one: $size_one" - return 1 - fi - if [[ $size_one -ne "$2" ]]; then - log 2 "Object one size mismatch ($size_one, $2)" - return 1 - fi - if ! key_two=$(echo "$objects" | jq -r '.Contents[1].Key' 2>&1); then - log 2 "error obtaining key two: $key_two" - return 1 - fi - if [[ $key_two != "$3" ]]; then - log 2 "Object two mismatch ($key_two, $3)" - return 1 - fi - if ! size_two=$(echo "$objects" | jq '.Contents[1].Size' 2>&1); then - log 2 "error obtaining size two: $size_two" - return 1 - fi - if [[ $size_two -ne "$4" ]]; then - log 2 "Object two size mismatch ($size_two, $4)" - return 1 - fi -} - -list_check_objects_v2() { - if ! check_param_count "list_check_objects_v2" "bucket, expected key one, expected size one, expected key two, expected size two" 5 $#; then - return 1 - fi - if ! list_objects_v2 "$1"; then - log 2 "error listing objects (s3api, v1)" - return 1 - fi - if ! check_listed_objects "$2" "$3" "$4" "$5"; then - log 2 "error checking listed objects" - return 1 - fi - return 0 -} - -list_check_objects_rest() { - if ! check_param_count "list_check_objects_rest" "bucket" 1 $#; then - return 1 - fi - - local response - if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then - log 2 "error listing and parsing objects: $response" - return 1 - fi - mapfile -t object_array <<< "$response" - - object_found=false - # shellcheck disable=SC2154 - for object in "${object_array[@]}"; do - log 5 "object: $object" - if [[ $object == "$test_file" ]]; then - object_found=true - break - fi - done - if [[ $object_found == "false" ]]; then - log 2 "object not found" - return 1 - fi - return 0 -} - -list_check_objects_common() { - if ! check_param_count "list_check_objects_common" "client, bucket, object one, object two" 4 $#; then - return 1 - fi - if ! list_objects "$1" "$2"; then - log 2 "error listing objects" - return 1 - fi - local object_one_found=false - local object_two_found=false - # shellcheck disable=SC2154 - for object in "${object_array[@]}"; do - if [ "$object" == "$3" ] || [ "$object" == "s3://$2/$3" ]; then - object_one_found=true - elif [ "$object" == "$4" ] || [ "$object" == "s3://$2/$4" ]; then - object_two_found=true - fi - done - - if [ $object_one_found != true ] || [ $object_two_found != true ]; then - log 2 "$3 and/or $4 not listed (all objects: ${object_array[*]})" - return 1 - fi - return 0 -} - -list_objects_check_file_count() { - if ! check_param_count "list_objects_check_file_count" "client, bucket, count" 3 $#; then - return 1 - fi - if ! list_objects "$1" "$2"; then - log 2 "error listing objects" - return 1 - fi - if [[ $LOG_LEVEL -ge 5 ]]; then - log 5 "Array: ${object_array[*]}" - fi - local file_count="${#object_array[@]}" - if [[ $file_count != "$3" ]]; then - log 2 "file count should be $3, is $file_count" - return 1 - fi - return 0 -} - -check_object_listing_with_prefixes() { - if ! check_param_count "check_object_listing_with_prefixes" "bucket, folder name, object name" 3 $#; then - return 1 - fi - if ! list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"; then - log 2 "error listing objects with delimiter '/'" - return 1 - fi - if ! prefix=$(echo "${objects[@]}" | jq -r ".CommonPrefixes[0].Prefix" 2>&1); then - log 2 "error getting object prefix from object list: $prefix" - return 1 - fi - if [[ $prefix != "$2/" ]]; then - log 2 "prefix doesn't match (expected $2, actual $prefix/)" - return 1 - fi - if ! list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#"; then - log 2 "error listing objects with delimiter '#" - return 1 - fi - if ! key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1); then - log 2 "error getting key from object list: $key" - return 1 - fi - if [[ $key != "$2/$3" ]]; then - log 2 "key doesn't match (expected $key, actual $2/$3)" - return 1 - fi - return 0 -} - -list_objects_with_user_rest_verify_access_denied() { - if ! check_param_count "list_objects_with_user_rest_verify_access_denied" "bucket, username, password" 3 $#; then - return 1 - fi - if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then - log 2 "error attempting to list objects: $result" - return 1 - fi - if [ "$result" != "403" ]; then - log 2 "expected response code of '403', was '$result'" - return 1 - fi - error_message="$(cat "$TEST_FILE_FOLDER/objects.txt")" - if [[ "$error_message" != *"Access Denied"* ]]; then - log 2 "unexpected error message: $error_message" - return 1 - fi - return 0 -} - -list_objects_with_user_rest_verify_success() { - if ! check_param_count "list_objects_with_user_rest_verify_success" "bucket, username, password, expected object" 4 $#; then - return 1 - fi - if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then - log 2 "error attempting to list objects: $result" - return 1 - fi - if [ "$result" != "200" ]; then - log 2 "expected response code of '200', was '$result' (error: $(cat "$TEST_FILE_FOLDER/objects.txt"))" - return 1 - fi - if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then - log 2 "error getting object key: $key" - return 1 - fi - if [ "$key" != "$4" ]; then - log 2 "expected '$4', was '$key'" - return 1 - fi - return 0 -} - -list_objects_check_params_get_token() { - if ! check_param_count "list_objects_check_params_get_token" "bucket, files, version two" 4 $#; then - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="$4" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then - log 2 "error attempting to get bucket ACL response: $result" - return 1 - fi - if [ "$result" != "200" ]; then - log 2 "expected '200' was '$result' ($(cat "$TEST_FILE_FOLDER/objects.txt"))" - return 1 - fi - log 5 "objects: $(cat "$TEST_FILE_FOLDER/objects.txt")" - if ! list_bucket_result=$(xmllint --xpath '//*[local-name()="ListBucketResult"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then - log 2 "error getting list bucket result: $list_bucket_result" - return 1 - fi - echo -n "$list_bucket_result" > "$TEST_FILE_FOLDER/list_bucket_result.txt" - if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "$2" "Key"; then - log 2 "key mismatch" - return 1 - fi - if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "1" "MaxKeys"; then - log 2 "max keys mismatch" - return 1 - fi - if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "1" "KeyCount"; then - log 2 "key count mismatch" - return 1 - fi - if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "true" "IsTruncated"; then - log 2 "key count mismatch" - return 1 - fi - if ! continuation_token=$(xmllint --xpath '//*[local-name()="NextContinuationToken"]/text()' <(echo "$list_bucket_result") 2>&1); then - log 2 "error getting next continuation token: $continuation_token" - return 1 - fi - echo "$continuation_token" - return 0 -} - -list_objects_check_continuation_error() { - if ! check_param_count "list_objects_check_continuation_error" "bucket, continuation token" 2 $#; then - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 CONTINUATION_TOKEN="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then - log 2 "error attempting to get bucket ACL response: $result" - return 1 - fi - if [ "$result" != "400" ]; then - log 2 "expected result code of '400' was '$result'" - return 1 - fi - if ! check_xml_element "$TEST_FILE_FOLDER/objects.txt" "InvalidArgument" "Error" "Code"; then - log 2 "invalid error code" - return 1 - fi -} - -list_objects_v1_check_nextmarker_empty() { - if ! check_param_count "list_objects_v1_check_nextmarker_empty" "bucket" 1 $#; then - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then - log 2 "error attempting to get bucket ACL response: $result" - return 1 - fi - log 5 "output: $(cat "$TEST_FILE_FOLDER/objects.txt")" - if ! next_marker=$(xmllint --xpath '//*[local-name()="NextMarker"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then - if [[ "$next_marker" != *"XPath set is empty"* ]]; then - log 2 "unexpected error: $next_marker" - return 1 - fi - return 0 - fi - log 5 "next marker: $next_marker" - marker_text=$(xmllint --xpath 'string(/NextMarker)' <(echo "$next_marker") 2>&1) - log 5 "marker text: $marker_text" - if [[ "$marker_text" != *"Document is empty"* ]]; then - log 2 "NextMarker text should be empty, but is $marker_text" - return 1 - fi - return 0 -} - -get_delete_marker_and_verify_405() { - if ! check_param_count "get_delete_marker_and_verify_405" "bucket, file name" 2 $#; then - return 1 - fi - if ! response=$(list_object_versions_rest "$1" 2>&1); then - log 2 "error listing REST object versions" - return 1 - fi - versions_file="$response" - log 5 "versions: $(cat "$versions_file")" - - if ! version_id=$(xmllint --xpath "//*[local-name()=\"DeleteMarker\"]/*[local-name()=\"VersionId\"]/text()" "$versions_file" 2>&1); then - log 2 "error getting XML value: $version_id" - return 1 - fi - log 5 "xml val: $version_id" - - if ! response=$(get_file_name 2>&1); then - log 2 "error getting file name: $response" - return 1 - fi - if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/$response" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then - log 2 "error getting result: $result" - return 1 - fi - if [ "$result" != "405" ]; then - log 2 "expected '405', was '$result' ($(cat "$TEST_FILE_FOLDER/$response"))" - return 1 - fi - return 0 -}