mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-18 21:26:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2140 from versity/test/util_list_objects

Browse Source 
test: ListObjects util code cleanup, multipart updates
This commit is contained in:
Ben McClelland
2026-05-21 08:14:21 -07:00
committed by GitHub
parent 6fd0b1395e 48f79a6171
commit eade1e3a71
17 changed files with 645 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/commands/list_objects.sh
+21 -17
View File
@@ -1,8 +1,5 @@
#!/usr/bin/env bash
source ./tests/util/util_list_objects.sh
source ./tests/commands/command.sh
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
@@ -17,6 +14,8 @@ source ./tests/commands/command.sh
# specific language governing permissions and limitations
# under the License.
source ./tests/commands/command.sh
# args: client, bucket name
# return 0 if able to list, 1 if not
list_objects() {
@@ -48,15 +47,13 @@ list_objects() {
return 1
fi
object_array=()
while IFS= read -r line; do
if [[ $line != *InsecureRequestWarning* ]]; then
object_name=$(echo "$line" | awk '{print $NF}')
object_array+=("$object_name")
printf '%s\n' "$line"
fi
done <<< "$output"
export object_array
return 0
}
# args: bucket name
@@ -74,13 +71,12 @@ list_objects_s3api() {
log 5 "list_objects_s3api: raw data returned: $output"
modified_output=$(echo "$output" | grep -v "InsecureRequestWarning")
object_array=()
log 5 "modified output: $modified_output"
if echo "$modified_output" | jq -e 'has("Contents")'; then
if echo "$modified_output" | jq -e 'has("Contents")' >/dev/null; then
contents=$(echo "$modified_output" | jq -r '.Contents[]')
log 5 "contents: $contents"
keys=$(echo "$contents" | jq -r '.Key')
IFS=$'\n' read -rd '' -a object_array <<<"$keys"
printf '%s\n' "$keys"
fi
return 0
}
@@ -89,20 +85,25 @@ list_objects_s3api() {
# param: bucket
# export objects on success, return 1 for failure
list_objects_s3api_v1() {
if [ $# -lt 1 ] || [ $# -gt 2 ]; then
log 2 "list objects command requires bucket, (optional) delimiter"
if ! check_param_count_ge_le "bucket, delimiter (optional)" 1 2 $#; then
return 1
fi
local result=0
if [ "$2" == "" ]; then
objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1") || local result=$?
objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" 2>&1) || result=$?
else
objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2") || local result=$?
objects=$(send_command aws --no-verify-ssl s3api list-objects --bucket "$1" --delimiter "$2" 2>&1) || result=$?
fi
if [[ $result -ne 0 ]]; then
log 2 "error listing objects: $objects"
return 1
fi
export objects
while IFS= read -r line; do
if [[ $line != *InsecureRequestWarning* ]]; then
printf '%s\n' "$line"
fi
done <<< "$objects"
return 0
}
list_objects_with_prefix() {
@@ -126,8 +127,11 @@ list_objects_with_prefix() {
log 2 "error listing objects: $objects"
return 1
fi
log 5 "output: $objects"
export objects
while IFS= read -r line; do
if [[ $line != *InsecureRequestWarning* ]]; then
printf '%s\n' "$line"
fi
done <<< "$objects"
return 0
}
tests/commands/list_objects_v2.sh
+9 -2
View File
@@ -22,15 +22,22 @@ list_objects_v2() {
log 2 "list objects command missing bucket and/or path"
return 1
fi
objects=$(send_command aws --no-verify-ssl s3api list-objects-v2 --bucket "$1") || local result=$?
local result=0
objects=$(send_command aws --no-verify-ssl s3api list-objects-v2 --bucket "$1" 2>&1) || result=$?
if [[ $result -ne 0 ]]; then
log 2 "error listing objects: $objects"
return 1
fi
while IFS= read -r line; do
if [[ $line != *InsecureRequestWarning* ]]; then
printf '%s\n' "$line"
fi
done <<< "$objects"
return 0
}
list_objects_v2_rest_callback() {
if ! check_param_count_gt "bucket, expected response code, callback fn, params" 3 $#; then
if ! check_param_count_gt "bucket, expected response code, callback fn, params, --, callback params" 3 $#; then
return 1
fi
if ! send_rest_go_command_callback "$2" "$3" "-bucketName" "$1" "-method" "GET" "-query" "list-type=2" "${@:4}"; then
tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh
+78 -38
View File
@@ -15,16 +15,17 @@
# under the License.
calculate_multipart_checksum() {
if ! check_param_count_gt "checksum type, part count, data file, checksums" 4 $#; then
if ! check_param_count_gt "checksum type, part count, data file, lowercase algorithm, checksums" 4 $#; then
return 1
fi
log 5 "checksums: ${*:4}"
log 5 "checksums: ${*:5}"
if [ "$1" == "COMPOSITE" ]; then
if ! calculate_composite_checksum "$lowercase_checksum_algorithm" "${@:4}"; then
log 2 "error calculating checksum"
if ! response=$(calculate_composite_checksum "$4" "${@:5}" 2>&1); then
log 2 "error calculating checksum: $response"
return 1
fi
checksum="$composite-$2"
checksum="$response-$2"
echo "$checksum"
return 0
fi
@@ -32,42 +33,39 @@ calculate_multipart_checksum() {
log 2 "unrecognized checksum type: $1"
return 1
fi
if ! checksum=$(DATA_FILE="$3" CHECKSUM_TYPE="$lowercase_checksum_algorithm" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
log 2 "error calculating checksum: $checksum"
if ! response=$(DATA_FILE="$3" CHECKSUM_TYPE="$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
log 2 "error calculating checksum: $response"
return 1
fi
echo "$response"
return 0
}
complete_multipart_upload_with_checksum() {
if ! check_param_count_v2 "bucket, key, file, upload ID, part count, checksum type, checksum algorithm" 7 $#; then
if ! check_param_count_v2 "bucket, key, file, upload ID, part count, parts payload, checksum type, checksum algorithm" 8 $#; then
return 1
fi
log 5 "checksum algorithm: $7"
lowercase_checksum_algorithm=$(echo -n "$7" | tr '[:upper:]' '[:lower:]')
if ! upload_parts_rest_with_checksum_before_completion "$1" "$2" "$3" "$4" "$5" "$lowercase_checksum_algorithm"; then
log 2 "error uploading REST parts with checksum before completion"
log 5 "checksum algorithm: $8"
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
log 5 "parts payload: $parts_payload"
log 5 "checksums: ${checksums[*]}"
if ! calculate_multipart_checksum "$6" "$5" "$3" "${checksums[@]}"; then
log 2 "error calculating multipart checksum"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$4" PARTS="$parts_payload" CHECKSUM_TYPE="$6" CHECKSUM_ALGORITHM="$7" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then
response_file="$response"
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$4" PARTS="$6" CHECKSUM_TYPE="$7" CHECKSUM_ALGORITHM="$8" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/$response_file" ./tests/rest_scripts/complete_multipart_upload.sh); then
log 2 "error completing multipart upload"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/$response_file"))"
return 1
fi
log 5 "result: $(cat "$TEST_FILE_FOLDER/result.txt")"
log 5 "result: $(cat "$TEST_FILE_FOLDER/$response_file")"
return 0
}
calculate_composite_checksum() {
log 5 "algorithm: $1, checksums: ${*:2}"
if ! check_param_count_gt "algorithm, at least two checksums" 3 $#; then
return 1
fi
@@ -96,6 +94,7 @@ calculate_composite_checksum() {
fi
log 5 "composite: $composite"
echo "$composite"
return 0
}
test_multipart_upload_with_checksum() {
@@ -103,21 +102,29 @@ test_multipart_upload_with_checksum() {
if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then
return 1
fi
if ! file_and_bucket=$(setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
if ! file_and_bucket=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
log 2 "error setting up file and bucket"
return 1
fi
read -r bucket_name mp_file_name <<< "$file_and_bucket"
log 5 "file name: $mp_file_name, file info: $(ls -l "$TEST_FILE_FOLDER/$mp_file_name")"
if ! perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2"; then
log 2 "error performing multipart upload with checksum before completion"
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then
log 2 "error performing multipart upload with checksum before completion: $response"
return 1
fi
if ! calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "${checksums[@]}"; then
mapfile -t response_lines<<< "$response"
local lowercase_checksum_algorithm="${response_lines[0]}"
local upload_id="${response_lines[1]}"
local checksum_string="${response_lines[2]}"
local payload_parts="${response_lines[3]}"
read -r -a checksums <<< "$checksum_string"
if ! calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}"; then
log 2 "error calculating multipart checksum"
return 1
fi
if ! complete_multipart_upload_with_checksum "$bucket_name" "$mp_file_name" "$TEST_FILE_FOLDER/$mp_file_name" "$upload_id" 2 "$1" "$2"; then
if ! complete_multipart_upload_with_checksum "$bucket_name" "$mp_file_name" "$TEST_FILE_FOLDER/$mp_file_name" "$upload_id" 2 "$payload_parts" "$1" "$2"; then
log 2 "error completing multipart upload with checksum"
return 1
fi
@@ -128,16 +135,19 @@ test_complete_multipart_upload_unneeded_algorithm_parameter() {
if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then
return 1
fi
if ! file_and_bucket=$(setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
if ! file_and_bucket=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
log 2 "error setting up file and bucket"
return 1
fi
read -r bucket_name mp_file_name <<< "$file_and_bucket"
if ! perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2"; then
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then
log 2 "error performing multipart upload with checksum before completion"
return 1
fi
log 5 "upload ID: $upload_id"
mapfile -t response_lines <<< "$response"
local upload_id="${response_lines[1]}"
local parts_payload="${response_lines[3]}"
if ! complete_multipart_upload_rest_nonexistent_param "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload"; then
log 2 "error completing multipart upload with nonexistent param"
return 1
@@ -149,21 +159,30 @@ test_complete_multipart_upload_incorrect_checksum() {
if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then
return 1
fi
if ! file_and_bucket=$(setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
if ! response=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
log 2 "error setting up file and bucket"
return 1
fi
read -r bucket_name mp_file_name <<< "$file_and_bucket"
if ! perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2"; then
read -r bucket_name mp_file_name <<< "$response"
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then
log 2 "error performing multipart upload with checksum before completion"
return 1
fi
if ! calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "${checksums[@]}"; then
mapfile -t response_lines<<< "$response"
local lowercase_checksum_algorithm="${response_lines[0]}"
local upload_id="${response_lines[1]}"
local checksum_string="${response_lines[2]}"
local payload_parts="${response_lines[3]}"
read -r -a checksums <<< "$checksum_string"
if ! response=$(calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}" 2>&1); then
log 2 "error calculating multipart checksum"
return 1
fi
if ! complete_multipart_upload_rest_incorrect_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload" "$1" "$2" "$checksum"; then
log 2 "error completing multipart upload with nonexistent param"
checksum="$response"
if ! complete_multipart_upload_rest_incorrect_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$payload_parts" "$1" "$2" "$checksum"; then
log 2 "error completing multipart upload with incorrect checksum"
return 1
fi
return 0
@@ -178,12 +197,16 @@ test_complete_multipart_upload_invalid_checksum() {
return 1
fi
read -r bucket_name mp_file_name <<< "$file_and_bucket"
if ! perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2"; then
log 2 "error performing multipart upload with checksum before completion"
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then
log 2 "error performing multipart upload with checksum before completion: $response"
return 1
fi
mapfile -t response_lines <<< "$response"
local upload_id="${response_lines[1]}"
local parts_payload="${response_lines[3]}"
if ! complete_multipart_upload_rest_invalid_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload" "$1" "$2" "wrong"; then
log 2 "error completing multipart upload with nonexistent param"
log 2 "error completing multipart upload with invalid checksum"
return 1
fi
return 0
@@ -270,3 +293,20 @@ perform_multipart_upload_rest_variable_parts() {
fi
return 0
}
complete_multipart_upload_empty_upload_id() {
if ! check_param_count_v2 "bucket, key, file" 3 $#; then
return 1
fi
local response upload_id parts_payload
if ! response=$(multipart_upload_rest_before_completion "$1" "$2" "$3" 2 2>&1); then
log 2 "error performing multipart upload before completion: $response"
return 1
fi
read -r upload_id parts_payload <<< "$response"
if ! complete_multipart_upload_rest_expect_error "$1" "$2" "" "$parts_payload" "" "404" "NoSuchUpload" "The specified upload does not exist"; then
log 2 "error completing multipart upload"
return 1
fi
return 0
}
tests/drivers/head_object/head_object_rest.sh
+32
View File
@@ -254,3 +254,35 @@ check_header_partial_content_response() {
fi
return 0
}
get_delete_marker_and_verify_405() {
if ! check_param_count "get_delete_marker_and_verify_405" "bucket, file name" 2 $#; then
return 1
fi
if ! response=$(list_object_versions_rest "$1" 2>&1); then
log 2 "error listing REST object versions"
return 1
fi
versions_file="$response"
log 5 "versions: $(cat "$versions_file")"
if ! version_id=$(xmllint --xpath "//*[local-name()=\"DeleteMarker\"]/*[local-name()=\"VersionId\"]/text()" "$versions_file" 2>&1); then
log 2 "error getting XML value: $version_id"
return 1
fi
log 5 "xml val: $version_id"
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/$response" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then
log 2 "error getting result: $result"
return 1
fi
if [ "$result" != "405" ]; then
log 2 "expected '405', was '$result' ($(cat "$TEST_FILE_FOLDER/$response"))"
return 1
fi
return 0
}
tests/drivers/list_objects/list_objects.sh
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env bash
# Copyright 2026 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
list_check_objects_common() {
if ! check_param_count "list_check_objects_common" "client, bucket, object one, object two" 4 $#; then
return 1
fi
if ! response=$(list_objects "$1" "$2" 2>&1); then
log 2 "error listing objects: $response"
return 1
fi
log 5 "response: $response"
mapfile -t object_list <<< "$response"
local object_one_found=false
local object_two_found=false
for object_data in "${object_list[@]}"; do
if [ "$1" == "s3cmd" ]; then
object="$(echo -n "$object_data" | awk '{ for (i=4; i<=NF; i++) printf "%s%s", $i, (i<NF ? OFS : ORS) }')"
else
object="$object_data"
fi
if [ "$object" == "$3" ] || [ "$object" == "s3://$2/$3" ]; then
object_one_found=true
elif [ "$object" == "$4" ] || [ "$object" == "s3://$2/$4" ]; then
object_two_found=true
fi
done
if [ $object_one_found != true ] || [ $object_two_found != true ]; then
log 2 "$3 and/or $4 not listed (all objects: ${object_list[*]})"
return 1
fi
return 0
}
list_objects_check_file_count() {
if ! check_param_count "list_objects_check_file_count" "client, bucket, count" 3 $#; then
return 1
fi
if ! response=$(list_objects "$1" "$2" 2>&1); then
log 2 "error listing objects: $response"
return 1
fi
mapfile -t object_array <<< "$response"
if [[ $LOG_LEVEL -ge 5 ]]; then
log 5 "Array: ${object_array[*]}"
fi
local file_count="${#object_array[@]}"
if [[ $file_count != "$3" ]]; then
log 2 "file count should be $3, is $file_count"
return 1
fi
return 0
}
tests/drivers/list_objects/list_objects_rest.sh
+216 -3
View File
@@ -56,14 +56,15 @@ list_objects_success_or_access_denied() {
}
check_v2_objects() {
if ! check_param_count_v2 "data file" 1 $#; then
if ! check_param_count_gt "data file, expected objects" 1 $#; then
return 1
fi
object_count=$(($#-1))
if ! check_xml_element "$1" "$object_count" "ListBucketResult" "KeyCount"; then
log 2 "error checking KeyCount element"
return 1
fi
for object in "${expected_objects[@]}"; do
for object in "${@:2}"; do
if ! check_if_element_exists "$1" "$object" "ListBucketResult" "Contents" "Key"; then
log 2 "error checking if element '$object' exists"
return 1
@@ -78,7 +79,7 @@ list_check_objects_rest_v2() {
fi
object_count=$2
expected_objects=("${@:3:$object_count}")
if ! list_objects_v2_rest_callback "$1" "200" "check_v2_objects" "${@:((3+$object_count))}"; then
if ! list_objects_v2_rest_callback "$1" "200" "check_v2_objects" "${@:((3+$object_count))}" "--" "${expected_objects[@]}"; then
log 2 "error sending list objects v2 command and checking callback"
return 1
fi
@@ -420,3 +421,215 @@ list_objects_delimiter() {
run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "2" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt"
assert_success
}
parse_objects_list_rest() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
local response
if ! response=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then
if [[ "$response" == *"XPath set is empty"* ]]; then
return 0
fi
log 2 "error getting object list: $response"
return 1
else
object_list="$response"
fi
log 5 "object list: '$object_list'"
while IFS= read -r object; do
log 5 "parsed key: '$object'"
unescaped_object="$(echo -n "$object" | xmlstarlet unesc)"
echo "$unescaped_object"
done <<< "$object_list"
return 0
}
list_check_objects_rest() {
if ! check_param_count "list_check_objects_rest" "bucket" 1 $#; then
return 1
fi
local response
if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then
log 2 "error listing and parsing objects: $response"
return 1
fi
mapfile -t object_array <<< "$response"
object_found=false
# shellcheck disable=SC2154
for object in "${object_array[@]}"; do
log 5 "object: $object"
if [[ $object == "$test_file" ]]; then
object_found=true
break
fi
done
if [[ $object_found == "false" ]]; then
log 2 "object not found"
return 1
fi
return 0
}
list_objects_with_user_rest_verify_access_denied() {
if ! check_param_count "list_objects_with_user_rest_verify_access_denied" "bucket, username, password" 3 $#; then
return 1
fi
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to list objects: $result"
return 1
fi
if [ "$result" != "403" ]; then
log 2 "expected response code of '403', was '$result'"
return 1
fi
error_message="$(cat "$TEST_FILE_FOLDER/$file_name")"
if [[ "$error_message" != *"Access Denied"* ]]; then
log 2 "unexpected error message: $error_message"
return 1
fi
return 0
}
list_objects_with_user_rest_verify_success() {
if ! check_param_count "list_objects_with_user_rest_verify_success" "bucket, username, password, expected object" 4 $#; then
return 1
fi
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to list objects: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected response code of '200', was '$result' (error: $(cat "$TEST_FILE_FOLDER/$file_name"))"
return 1
fi
if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/$file_name" 2>&1); then
log 2 "error getting object key: $key"
return 1
fi
if [ "$key" != "$4" ]; then
log 2 "expected '$4', was '$key'"
return 1
fi
return 0
}
list_objects_check_params_get_token() {
if ! check_param_count "list_objects_check_params_get_token" "bucket, files, version two" 4 $#; then
return 1
fi
if ! response=$(get_file_names 2 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
read -r file_name list_bucket_file <<< "$response"
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="$4" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to get bucket ACL response: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200' was '$result' ($(cat "$TEST_FILE_FOLDER/$file_name"))"
return 1
fi
log 5 "objects: $(cat "$TEST_FILE_FOLDER/$file_name")"
if ! list_bucket_result=$(xmllint --xpath '//*[local-name()="ListBucketResult"]' "$TEST_FILE_FOLDER/$file_name" 2>&1); then
log 2 "error getting list bucket result: $list_bucket_result"
return 1
fi
echo -n "$list_bucket_result" > "$TEST_FILE_FOLDER/$list_bucket_file"
if ! check_xml_element "$TEST_FILE_FOLDER/$list_bucket_file" "$2" "Key"; then
log 2 "key mismatch"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/$list_bucket_file" "1" "MaxKeys"; then
log 2 "max keys mismatch"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/$list_bucket_file" "1" "KeyCount"; then
log 2 "key count mismatch"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/$list_bucket_file" "true" "IsTruncated"; then
log 2 "key count mismatch"
return 1
fi
if ! continuation_token=$(xmllint --xpath '//*[local-name()="NextContinuationToken"]/text()' "$TEST_FILE_FOLDER/$list_bucket_file" 2>&1); then
log 2 "error getting next continuation token: $continuation_token"
return 1
fi
echo "$continuation_token"
return 0
}
list_objects_check_continuation_error() {
if ! check_param_count "list_objects_check_continuation_error" "bucket, continuation token" 2 $#; then
return 1
fi
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 CONTINUATION_TOKEN="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to get bucket ACL response: $result"
return 1
fi
if [ "$result" != "400" ]; then
log 2 "expected result code of '400' was '$result'"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/$file_name" "InvalidArgument" "Error" "Code"; then
log 2 "invalid error code"
return 1
fi
return 0
}
list_objects_v1_check_nextmarker_empty() {
if ! check_param_count "list_objects_v1_check_nextmarker_empty" "bucket" 1 $#; then
return 1
fi
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to get bucket ACL response: $result"
return 1
fi
log 5 "output: $(cat "$TEST_FILE_FOLDER/$file_name")"
if ! next_marker=$(xmllint --xpath '//*[local-name()="NextMarker"]' "$TEST_FILE_FOLDER/$file_name" 2>&1); then
if [[ "$next_marker" != *"XPath set is empty"* ]]; then
log 2 "unexpected error: $next_marker"
return 1
fi
return 0
fi
log 5 "next marker: $next_marker"
marker_text=$(xmllint --xpath 'string(/NextMarker)' <(echo "$next_marker") 2>&1)
log 5 "marker text: $marker_text"
if [[ "$marker_text" != *"Document is empty"* ]]; then
log 2 "NextMarker text should be empty, but is $marker_text"
return 1
fi
return 0
}
tests/drivers/list_objects/list_objects_s3api.sh
+136
View File
@@ -0,0 +1,136 @@
#!/usr/bin/env bash
# Copyright 2026 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
list_check_objects_v1() {
if ! check_param_count_gt "bucket, expected keys" 1 $#; then
return 1
fi
if ! response=$(list_objects_s3api_v1 "$1" 2>&1); then
log 2 "error listing objects (s3api, v1): $response"
return 1
fi
object_json="$response"
log 5 "object JSON: $response"
if ! check_listed_objects "$object_json" "${@:2}"; then
log 2 "error checking listed objects v2"
return 1
fi
return 0
}
check_listed_objects() {
if ! check_param_count_gt "JSON data, expected keys (if any)" 1 $#; then
return 1
fi
expected_key_count="$(($#-1))"
if ! response=$(echo "$1" | jq '.Contents | length' 2>&1); then
log 2 "error getting length: $response"
return 1
fi
key_count="$response"
if [ "$key_count" -ne "$expected_key_count" ]; then
log 2 "expected key count of '$expected_key_count', was '$key_count'"
return 1
fi
declare -A key_array size_array
for ((i=0; i<key_count; i++)); do
if ! response=$(echo "$1" | jq -r ".Contents[$i].Key" 2>&1); then
log 2 "error obtaining key one: $response"
return 1
fi
key="$response"
key_array["$key"]="true"
if ! response=$(echo "$1" | jq -r ".Contents[$i].Size" 2>&1); then
log 2 "error obtaining key one: $response"
return 1
fi
size_array["$key"]="$response"
done
for expected_key in "${@:2}"; do
if [ "${key_array[$expected_key]}" != "true" ]; then
continue
fi
if ! response=$(get_file_size "$TEST_FILE_FOLDER/$expected_key" 2>&1); then
log 2 "error getting file size: $response"
return 1
fi
if [ "${size_array[$expected_key]}" != "$response" ]; then
log 2 "size mismatch, expected '${size_array[$expected_key]}', was $response"
return 1
fi
unset key_array["$expected_key"]
done
if [ ${#key_array[@]} -ne 0 ]; then
log 2 "key mismatch: keys left over: '${!key_array[*]}'"
return 1
fi
return 0
}
list_check_objects_v2() {
if ! check_param_count_gt "bucket, expected keys" 1 $#; then
return 1
fi
if ! response=$(list_objects_v2 "$1" 2>&1); then
log 2 "error listing objects (s3api, v2): $response"
return 1
fi
object_json="$response"
if ! check_listed_objects "$object_json" "${@:2}"; then
log 2 "error checking listed objects"
return 1
fi
return 0
}
check_object_listing_with_prefixes() {
if ! check_param_count "check_object_listing_with_prefixes" "bucket, folder name, object name" 3 $#; then
return 1
fi
if ! response=$(list_objects_s3api_v1 "$1" "/" 2>&1); then
log 2 "error listing objects with delimiter '/': $response"
return 1
fi
object_json="$response"
if ! prefix=$(echo "$object_json" | jq -r ".CommonPrefixes[0].Prefix" 2>&1); then
log 2 "error getting object prefix from object list: $prefix"
return 1
fi
if [[ $prefix != "$2/" ]]; then
log 2 "prefix doesn't match (expected $2, actual $prefix/)"
return 1
fi
if ! response=$(list_objects_s3api_v1 "$1" "#" 2>&1); then
log 2 "error listing objects with delimiter '#: $response"
return 1
fi
objects="$response"
if ! key=$(echo "$objects" | jq -r ".Contents[0].Key" 2>&1); then
log 2 "error getting key from object list: $key"
return 1
fi
if [[ $key != "$2/$3" ]]; then
log 2 "key doesn't match (expected $key, actual $2/$3)"
return 1
fi
return 0
}
tests/drivers/rest.sh
+1
View File
@@ -14,6 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/drivers/list_objects/list_objects_rest.sh
source ./tests/drivers/xml.sh
check_rest_expected_error() {
tests/drivers/upload_part/upload_part_rest.sh
+10 -12
View File
@@ -85,6 +85,8 @@ upload_parts_rest_with_checksum_before_completion() {
log 5 "parts payload: $parts_payload"
done
log 5 "CHECKSUMS: ${checksums[*]}"
echo "${checksums[*]}"
echo "$parts_payload"
return 0
}
@@ -92,25 +94,21 @@ perform_full_multipart_upload_with_checksum_before_completion() {
if ! check_param_count_v2 "bucket, filename, checksum type, algorithm" 4 $#; then
return 1
fi
if ! bucket_name=$(get_bucket_name "$1" 2>&1); then
log 2 "error getting bucket name: $bucket_name"
return 1
fi
if ! setup_bucket_and_large_file_v2 "$bucket_name" "$2"; then
log 2 "error setting up bucket and large file"
return 1
fi
lowercase_checksum_algorithm=$(echo -n "$4" | tr '[:upper:]' '[:lower:]')
local response
if ! response=$(create_multipart_upload_rest "$bucket_name" "$2" "CHECKSUM_TYPE=$3 CHECKSUM_ALGORITHM=$4" "parse_upload_id" 2>&1); then
if ! response=$(create_multipart_upload_rest "$1" "$2" "CHECKSUM_TYPE=$3 CHECKSUM_ALGORITHM=$4" "parse_upload_id" 2>&1); then
log 2 "error creating multipart upload: $response"
return 1
fi
upload_id="$response"
lowercase_checksum_algorithm=$(echo -n "$4" | tr '[:upper:]' '[:lower:]')
if ! upload_parts_rest_with_checksum_before_completion "$bucket_name" "$2" "$TEST_FILE_FOLDER/$2" "$upload_id" 2 "$lowercase_checksum_algorithm"; then
log 2 "error uploading parts"
if ! response=$(upload_parts_rest_with_checksum_before_completion "$1" "$2" "$TEST_FILE_FOLDER/$2" "$upload_id" 2 "$lowercase_checksum_algorithm" 2>&1); then
log 2 "error uploading parts: $response"
return 1
fi
echo "$lowercase_checksum_algorithm"
echo "$upload_id"
echo "$response"
return 0
}
tests/test_rest_list_objects.sh
+2
View File
@@ -19,7 +19,9 @@ load ./bats-assert/load
source ./tests/setup.sh
source ./tests/drivers/create_bucket/create_bucket_rest.sh
source ./tests/drivers/list_objects/list_objects.sh
source ./tests/drivers/list_objects/list_objects_rest.sh
source ./tests/drivers/list_objects/list_objects_s3api.sh
# tags: curl,ListObjects,minimal-request
@test "test_rest_list_objects" {
tests/test_rest_multipart.sh
+12
View File
@@ -464,3 +464,15 @@ source ./tests/drivers/upload_part/upload_part_rest.sh
run complete_multipart_upload_invalid_object_size_string "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file"
assert_success
}
@test "REST - multipart - empty upload ID on completion" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/2146"
fi
run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME"
assert_success
read -r bucket_name file_name <<< "$output"
run complete_multipart_upload_empty_upload_id "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/$file_name"
assert_success
}
tests/test_s3_file_count.sh
+1
View File
@@ -17,6 +17,7 @@
load ./bats-support/load
load ./bats-assert/load
source ./tests/drivers/list_objects/list_objects.sh
source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
source ./tests/test_common.sh
tests/test_s3api_object.sh
+2
View File
@@ -48,6 +48,8 @@ source ./tests/drivers/get_object_legal_hold/get_object_legal_hold.sh
source ./tests/drivers/get_object_retention/get_object_retention_s3api.sh
source ./tests/drivers/get_object_tagging/get_object_tagging.sh
source ./tests/drivers/list_buckets/list_buckets_rest.sh
source ./tests/drivers/list_objects/list_objects.sh
source ./tests/drivers/list_objects/list_objects_s3api.sh
source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
source ./tests/drivers/file.sh
source ./tests/util/util_lock_config.sh
tests/test_s3api_root_inner.sh
+54 -52
View File
@@ -18,86 +18,90 @@ source ./tests/commands/delete_objects.sh
source ./tests/commands/list_objects_v2.sh
source ./tests/commands/list_parts.sh
source ./tests/drivers/put_object/put_object.sh
source ./tests/util/util_list_objects.sh
test_delete_objects_s3api_root() {
local object_one="test-file-one"
local object_two="test-file-two"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two"
run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2
assert_success
read -r bucket_name object_one object_two <<< "$output"
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$bucket_name" "$object_one"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$BUCKET_ONE_NAME" "$object_one"
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$bucket_name" "$object_two"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$BUCKET_ONE_NAME" "$object_two"
run delete_objects "$bucket_name" "$object_one" "$object_two"
assert_success
run delete_objects "$BUCKET_ONE_NAME" "$object_one" "$object_two"
assert_success
run object_exists "s3api" "$BUCKET_ONE_NAME" "$object_one"
run object_exists "s3api" "$bucket_name" "$object_one"
assert_failure 1
run object_exists "s3api" "$BUCKET_ONE_NAME" "$object_two"
run object_exists "s3api" "$bucket_name" "$object_two"
assert_failure 1
}
test_get_object_full_range_s3api_root() {
bucket_file="bucket_file"
run get_file_name
assert_success
bucket_file="$output"
echo -n "0123456789" > "$TEST_FILE_FOLDER/$bucket_file"
run setup_bucket "$BUCKET_ONE_NAME"
run setup_bucket_v3 "$BUCKET_ONE_NAME"
assert_success
bucket_name="$output"
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
run get_object_with_range "$BUCKET_ONE_NAME" "$bucket_file" "bytes=9-15" "$TEST_FILE_FOLDER/$bucket_file-range"
run get_object_with_range "$bucket_name" "$bucket_file" "bytes=9-15" "$TEST_FILE_FOLDER/$bucket_file-range"
assert_success
assert [ "$(cat "$TEST_FILE_FOLDER/$bucket_file-range")" == "9" ]
}
test_get_object_invalid_range_s3api_root() {
bucket_file="bucket_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file"
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
assert_success
read -r bucket_name bucket_file <<< "$output"
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
run get_object_with_range "$BUCKET_ONE_NAME" "$bucket_file" "bytes=0-0" "$TEST_FILE_FOLDER/$bucket_file-range"
run get_object_with_range "$bucket_name" "$bucket_file" "bytes=0-0" "$TEST_FILE_FOLDER/$bucket_file-range"
assert_success
}
test_put_object_s3api_root() {
bucket_file="bucket_file"
run get_file_name
assert_success
bucket_file="$output"
run create_test_files "$bucket_file"
assert_success
run setup_buckets "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
run setup_buckets_v3 "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
assert_success
read -r bucket_one bucket_two <<< "$output"
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$bucket_one" "$bucket_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
run copy_object "s3api" "$bucket_one/$bucket_file" "$bucket_two" "$bucket_file"
assert_success
run copy_object "s3api" "$BUCKET_ONE_NAME/$bucket_file" "$BUCKET_TWO_NAME" "$bucket_file"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/${bucket_file}_copy"
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_two" "$bucket_file" "$TEST_FILE_FOLDER/${bucket_file}_copy"
assert_success
}
test_get_object_attributes_s3api_root() {
bucket_file="bucket_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file"
run setup_bucket_and_file_v3 "$BUCKET_ONE_NAME"
assert_success
read -r bucket_name bucket_file <<< "$output"
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
run get_and_check_object_size "$BUCKET_ONE_NAME" "$bucket_file" 10
run get_and_check_object_size "$bucket_name" "$bucket_file" 10
assert_success
}
@@ -233,33 +237,31 @@ test_retention_bypass_s3api_root() {
}
test_s3api_list_objects_v1_s3api_root() {
local object_one="test-file-one"
local object_two="test-file-two"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two"
run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2
assert_success
read -r bucket_name object_one object_two <<< "$output"
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$bucket_name" "$object_one"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$BUCKET_ONE_NAME" "$object_one"
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$bucket_name" "$object_two"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$BUCKET_ONE_NAME" "$object_two"
assert_success
run list_check_objects_v1 "$BUCKET_ONE_NAME" "$object_one" 10 "$object_two" 10
run list_check_objects_v1 "$bucket_name" "$object_one" "$object_two"
assert_success
}
test_s3api_list_objects_v2_s3api_root() {
local object_one="test-file-one"
local object_two="test-file-two"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two"
run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2
assert_success
read -r bucket_name object_one object_two <<< "$output"
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$bucket_name" "$object_one"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$BUCKET_ONE_NAME" "$object_one"
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$bucket_name" "$object_two"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_two" "$BUCKET_ONE_NAME" "$object_two"
assert_success
run list_check_objects_v2 "$BUCKET_ONE_NAME" "$object_one" 10 "$object_two" 10
run list_check_objects_v2 "$bucket_name" "$object_one" "$object_two"
assert_success
}
tests/test_s3cmd.sh
+1
View File
@@ -25,6 +25,7 @@ source ./tests/commands/get_bucket_policy.sh
source ./tests/commands/put_bucket_policy.sh
source ./tests/drivers/get_bucket_location/get_bucket_location.sh
source ./tests/drivers/list_buckets/list_buckets.sh
source ./tests/drivers/list_objects/list_objects.sh
source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
source ./tests/util/util_object.sh
tests/test_s3cmd_file_count.sh
+1
View File
@@ -17,6 +17,7 @@
load ./bats-support/load
load ./bats-assert/load
source ./tests/drivers/list_objects/list_objects.sh
source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
source ./tests/test_common.sh
tests/util/util_list_objects.sh
-376
View File
@@ -1,376 +0,0 @@
#!/usr/bin/env bash
source ./tests/commands/list_objects_v2.sh
source ./tests/drivers/xml.sh
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
parse_objects_list_rest() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
local response
if ! response=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then
if [[ "$response" == *"XPath set is empty"* ]]; then
return 0
fi
log 2 "error getting object list: $response"
return 1
else
object_list="$response"
fi
log 5 "object list: '$object_list'"
while IFS= read -r object; do
log 5 "parsed key: '$object'"
unescaped_object="$(echo -n "$object" | xmlstarlet unesc)"
echo "$unescaped_object"
done <<< "$object_list"
return 0
}
list_check_objects_v1() {
if ! check_param_count "list_check_objects_v1" "bucket, expected key one, expected size one, expected key two, expected size two" 5 $#; then
return 1
fi
if ! list_objects_s3api_v1 "$1"; then
log 2 "error listing objects (s3api, v1)"
return 1
fi
if ! check_listed_objects "$2" "$3" "$4" "$5"; then
log 2 "error checking listed objects"
return 1
fi
return 0
}
check_listed_objects() {
if ! check_param_count "check_listed_objects" "expected key one, expected size one, expected key two, expected size two" 4 $#; then
return 1
fi
# shellcheck disable=SC2154
if ! key_one=$(echo "$objects" | jq -r '.Contents[0].Key' 2>&1); then
log 2 "error obtaining key one: $key_one"
return 1
fi
if [[ $key_one != "$1" ]]; then
log 2 "Object one mismatch ($key_one, $1)"
return 1
fi
if ! size_one=$(echo "$objects" | jq -r '.Contents[0].Size' 2>&1); then
log 2 "error obtaining size one: $size_one"
return 1
fi
if [[ $size_one -ne "$2" ]]; then
log 2 "Object one size mismatch ($size_one, $2)"
return 1
fi
if ! key_two=$(echo "$objects" | jq -r '.Contents[1].Key' 2>&1); then
log 2 "error obtaining key two: $key_two"
return 1
fi
if [[ $key_two != "$3" ]]; then
log 2 "Object two mismatch ($key_two, $3)"
return 1
fi
if ! size_two=$(echo "$objects" | jq '.Contents[1].Size' 2>&1); then
log 2 "error obtaining size two: $size_two"
return 1
fi
if [[ $size_two -ne "$4" ]]; then
log 2 "Object two size mismatch ($size_two, $4)"
return 1
fi
}
list_check_objects_v2() {
if ! check_param_count "list_check_objects_v2" "bucket, expected key one, expected size one, expected key two, expected size two" 5 $#; then
return 1
fi
if ! list_objects_v2 "$1"; then
log 2 "error listing objects (s3api, v1)"
return 1
fi
if ! check_listed_objects "$2" "$3" "$4" "$5"; then
log 2 "error checking listed objects"
return 1
fi
return 0
}
list_check_objects_rest() {
if ! check_param_count "list_check_objects_rest" "bucket" 1 $#; then
return 1
fi
local response
if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then
log 2 "error listing and parsing objects: $response"
return 1
fi
mapfile -t object_array <<< "$response"
object_found=false
# shellcheck disable=SC2154
for object in "${object_array[@]}"; do
log 5 "object: $object"
if [[ $object == "$test_file" ]]; then
object_found=true
break
fi
done
if [[ $object_found == "false" ]]; then
log 2 "object not found"
return 1
fi
return 0
}
list_check_objects_common() {
if ! check_param_count "list_check_objects_common" "client, bucket, object one, object two" 4 $#; then
return 1
fi
if ! list_objects "$1" "$2"; then
log 2 "error listing objects"
return 1
fi
local object_one_found=false
local object_two_found=false
# shellcheck disable=SC2154
for object in "${object_array[@]}"; do
if [ "$object" == "$3" ] || [ "$object" == "s3://$2/$3" ]; then
object_one_found=true
elif [ "$object" == "$4" ] || [ "$object" == "s3://$2/$4" ]; then
object_two_found=true
fi
done
if [ $object_one_found != true ] || [ $object_two_found != true ]; then
log 2 "$3 and/or $4 not listed (all objects: ${object_array[*]})"
return 1
fi
return 0
}
list_objects_check_file_count() {
if ! check_param_count "list_objects_check_file_count" "client, bucket, count" 3 $#; then
return 1
fi
if ! list_objects "$1" "$2"; then
log 2 "error listing objects"
return 1
fi
if [[ $LOG_LEVEL -ge 5 ]]; then
log 5 "Array: ${object_array[*]}"
fi
local file_count="${#object_array[@]}"
if [[ $file_count != "$3" ]]; then
log 2 "file count should be $3, is $file_count"
return 1
fi
return 0
}
check_object_listing_with_prefixes() {
if ! check_param_count "check_object_listing_with_prefixes" "bucket, folder name, object name" 3 $#; then
return 1
fi
if ! list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"; then
log 2 "error listing objects with delimiter '/'"
return 1
fi
if ! prefix=$(echo "${objects[@]}" | jq -r ".CommonPrefixes[0].Prefix" 2>&1); then
log 2 "error getting object prefix from object list: $prefix"
return 1
fi
if [[ $prefix != "$2/" ]]; then
log 2 "prefix doesn't match (expected $2, actual $prefix/)"
return 1
fi
if ! list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#"; then
log 2 "error listing objects with delimiter '#"
return 1
fi
if ! key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1); then
log 2 "error getting key from object list: $key"
return 1
fi
if [[ $key != "$2/$3" ]]; then
log 2 "key doesn't match (expected $key, actual $2/$3)"
return 1
fi
return 0
}
list_objects_with_user_rest_verify_access_denied() {
if ! check_param_count "list_objects_with_user_rest_verify_access_denied" "bucket, username, password" 3 $#; then
return 1
fi
if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to list objects: $result"
return 1
fi
if [ "$result" != "403" ]; then
log 2 "expected response code of '403', was '$result'"
return 1
fi
error_message="$(cat "$TEST_FILE_FOLDER/objects.txt")"
if [[ "$error_message" != *"Access Denied"* ]]; then
log 2 "unexpected error message: $error_message"
return 1
fi
return 0
}
list_objects_with_user_rest_verify_success() {
if ! check_param_count "list_objects_with_user_rest_verify_success" "bucket, username, password, expected object" 4 $#; then
return 1
fi
if ! result=$(AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to list objects: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected response code of '200', was '$result' (error: $(cat "$TEST_FILE_FOLDER/objects.txt"))"
return 1
fi
if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then
log 2 "error getting object key: $key"
return 1
fi
if [ "$key" != "$4" ]; then
log 2 "expected '$4', was '$key'"
return 1
fi
return 0
}
list_objects_check_params_get_token() {
if ! check_param_count "list_objects_check_params_get_token" "bucket, files, version two" 4 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="$4" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to get bucket ACL response: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200' was '$result' ($(cat "$TEST_FILE_FOLDER/objects.txt"))"
return 1
fi
log 5 "objects: $(cat "$TEST_FILE_FOLDER/objects.txt")"
if ! list_bucket_result=$(xmllint --xpath '//*[local-name()="ListBucketResult"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then
log 2 "error getting list bucket result: $list_bucket_result"
return 1
fi
echo -n "$list_bucket_result" > "$TEST_FILE_FOLDER/list_bucket_result.txt"
if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "$2" "Key"; then
log 2 "key mismatch"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "1" "MaxKeys"; then
log 2 "max keys mismatch"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "1" "KeyCount"; then
log 2 "key count mismatch"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "true" "IsTruncated"; then
log 2 "key count mismatch"
return 1
fi
if ! continuation_token=$(xmllint --xpath '//*[local-name()="NextContinuationToken"]/text()' <(echo "$list_bucket_result") 2>&1); then
log 2 "error getting next continuation token: $continuation_token"
return 1
fi
echo "$continuation_token"
return 0
}
list_objects_check_continuation_error() {
if ! check_param_count "list_objects_check_continuation_error" "bucket, continuation token" 2 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 CONTINUATION_TOKEN="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to get bucket ACL response: $result"
return 1
fi
if [ "$result" != "400" ]; then
log 2 "expected result code of '400' was '$result'"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/objects.txt" "InvalidArgument" "Error" "Code"; then
log 2 "invalid error code"
return 1
fi
}
list_objects_v1_check_nextmarker_empty() {
if ! check_param_count "list_objects_v1_check_nextmarker_empty" "bucket" 1 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
log 2 "error attempting to get bucket ACL response: $result"
return 1
fi
log 5 "output: $(cat "$TEST_FILE_FOLDER/objects.txt")"
if ! next_marker=$(xmllint --xpath '//*[local-name()="NextMarker"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then
if [[ "$next_marker" != *"XPath set is empty"* ]]; then
log 2 "unexpected error: $next_marker"
return 1
fi
return 0
fi
log 5 "next marker: $next_marker"
marker_text=$(xmllint --xpath 'string(/NextMarker)' <(echo "$next_marker") 2>&1)
log 5 "marker text: $marker_text"
if [[ "$marker_text" != *"Document is empty"* ]]; then
log 2 "NextMarker text should be empty, but is $marker_text"
return 1
fi
return 0
}
get_delete_marker_and_verify_405() {
if ! check_param_count "get_delete_marker_and_verify_405" "bucket, file name" 2 $#; then
return 1
fi
if ! response=$(list_object_versions_rest "$1" 2>&1); then
log 2 "error listing REST object versions"
return 1
fi
versions_file="$response"
log 5 "versions: $(cat "$versions_file")"
if ! version_id=$(xmllint --xpath "//*[local-name()=\"DeleteMarker\"]/*[local-name()=\"VersionId\"]/text()" "$versions_file" 2>&1); then
log 2 "error getting XML value: $version_id"
return 1
fi
log 5 "xml val: $version_id"
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/$response" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" VERSION_ID="$version_id" ./tests/rest_scripts/head_object.sh); then
log 2 "error getting result: $result"
return 1
fi
if [ "$result" != "405" ]; then
log 2 "expected '405', was '$result' ($(cat "$TEST_FILE_FOLDER/$response"))"
return 1
fi
return 0
}