mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-08 00:57:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: ListObjectsV2 query updates, file updates, bash cleanup

Browse Source
This commit is contained in:
Luke McCrone
2026-04-14 18:50:10 -03:00
parent 393477aafd
commit 1632c8c006
27 changed files with 844 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/commands/create_bucket.sh
+1 -2
View File
@@ -25,8 +25,7 @@ create_bucket() {
return 1
fi
local exit_code=0
local error
local exit_code=0 error
if [[ $1 == 's3' ]]; then
error=$(send_command aws --no-verify-ssl s3 mb s3://"$2" 2>&1) || exit_code=$?
elif [[ $1 == 's3api' ]]; then
tests/commands/head_object.sh
+3 -2
View File
@@ -59,10 +59,11 @@ head_object_rest_expect_success_callback() {
return 1
fi
env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2 $3"
if ! send_rest_command_expect_success_callback "$env_vars" "./tests/rest_scripts/head_object.sh" "200" "$4"; then
log 2 "error sending REST command and checking error"
if ! callback_result=$(send_rest_command_expect_success_callback "$env_vars" "./tests/rest_scripts/head_object.sh" "200" "$4" 2>&1); then
log 2 "error sending REST command and checking error: $callback_result"
return 1
fi
echo "$callback_result"
return 0
}
tests/commands/list_buckets.sh
+5 -2
View File
@@ -118,9 +118,12 @@ list_buckets_rest() {
if ! check_param_count_v2 "params, callback" 2 $#; then
return 1
fi
if ! send_rest_command_expect_success_callback "$1" "./tests/rest_scripts/list_buckets.sh" "200" "$2"; then
log 2 "error sending REST command and checking error"
if ! callback_result=$(send_rest_command_expect_success_callback "$1" "./tests/rest_scripts/list_buckets.sh" "200" "$2" 2>&1); then
log 2 "error sending REST command and checking error: $callback_result"
return 1
else
buckets="$callback_result"
fi
echo "$buckets"
return 0
}
tests/commands/upload_part_copy.sh
+1 -1
View File
@@ -20,7 +20,7 @@ upload_part_copy() {
return 1
fi
local etag_json
log 5 "parameters: $1 $2 $3 $4 $5"
log 5 "parameters: '$1' '$2' '$3' '$4' '$5'"
etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$?
if [[ $uploaded -ne 0 ]]; then
log 2 "Error uploading part $5: $etag_json"
tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh
+4 -2
View File
@@ -193,10 +193,12 @@ complete_multipart_upload_invalid_object_size_string() {
if ! check_param_count_v2 "bucket, key, file" 3 $#; then
return 1
fi
if ! multipart_upload_rest_before_completion "$1" "$2" "$3" 2; then
log 2 "error performing multipart upload before completion"
local response upload_id parts_payload
if ! response=$(multipart_upload_rest_before_completion "$1" "$2" "$3" 2 2>&1); then
log 2 "error performing multipart upload before completion: $response"
return 1
fi
read -r upload_id parts_payload <<< "$response"
if ! complete_multipart_upload_rest_expect_error "$1" "$2" "$upload_id" "$parts_payload" "MULTIPART_OBJECT_SIZE=size" "400" "InvalidRequest" "Value for x-amz-mp-object-size header is invalid"; then
log 2 "error completing multipart upload"
return 1
tests/drivers/delete_bucket/delete_bucket_rest.sh
+4 -4
View File
@@ -29,12 +29,12 @@ delete_buckets_with_prefix() {
log 2 "delete_buckets_with_prefix requires non-empty prefix"
return 1
fi
if ! list_buckets_rest "PREFIX=$1" "parse_bucket_list"; then
log 2 "error listing buckets with prefix"
local response
if ! response=$(list_buckets_rest "PREFIX=$1" "parse_bucket_list" 2>&1); then
log 2 "error listing buckets with prefix: $response"
return 1
fi
# shellcheck disable=SC2154
log 5 "buckets: ${bucket_array[*]}"
read -r -a bucket_array <<< "$response"
for bucket in "${bucket_array[@]}"; do
if ! delete_bucket_recursive "$bucket"; then
log 2 "error with recursive bucket delete of bucket '$bucket'"
tests/drivers/delete_object/delete_object_rest.sh
+22 -10
View File
@@ -24,13 +24,16 @@ list_and_delete_objects() {
if ! check_param_count "list_and_delete_objects" "bucket" 1 $#; then
return 1
fi
if ! list_objects_rest "$1" "parse_objects_list_rest"; then
log 2 "error getting object list"
local response
if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then
log 2 "error getting object list: $response"
return 1
fi
# shellcheck disable=SC2154
log 5 "objects: ${object_array[*]}"
mapfile -t object_array <<< "$response"
for object in "${object_array[@]}"; do
if [ "$object" == "" ]; then
break
fi
if ! clear_object_in_bucket "$1" "$object"; then
log 2 "error deleting object $object"
return 1
@@ -114,11 +117,16 @@ put_object_with_lock_mode_and_delete_latest_version() {
log 2 "error sending put object command with object lock"
return 1
fi
if ! send_rest_go_command_callback "200" "parse_latest_version_id" \
"-method" "GET" "-bucketName" "$2" "-query" "versions="; then
log 2 "error checking versions before deletion"
local response
if ! response=$(send_rest_go_command_callback "200" "parse_latest_version_id" \
"-method" "GET" "-bucketName" "$2" "-query" "versions=" 2>&1); then
log 2 "error checking versions before deletion: $response"
return 1
else
version_id="$response"
fi
log 5 "version ID: $version_id"
if ! delete_object_version_rest_expect_error "$2" "$3" "$version_id" "403" "AccessDenied" "object protected by object lock"; then
log 2 "shouldn't have been able to delete"
return 1
@@ -140,11 +148,15 @@ attempt_to_delete_version_after_retention_policy() {
log 2 "error sending put object command"
return 1
fi
if ! send_rest_go_command_callback "200" "parse_latest_version_id" \
"-method" "GET" "-bucketName" "$2" "-query" "versions="; then
log 2 "error checking versions before deletion"
local response
if ! response=$(send_rest_go_command_callback "200" "parse_latest_version_id" \
"-method" "GET" "-bucketName" "$2" "-query" "versions=" 2>&1); then
log 2 "error checking versions before deletion: $response"
return 1
fi
version_id="$response"
if ! delete_object_version_rest_expect_error "$2" "$3" "$version_id" "403" "AccessDenied" "object protected by object lock"; then
log 2 "shouldn't have been able to delete"
return 1
tests/drivers/file.sh
+121 -29
View File
@@ -163,15 +163,25 @@ setup_bucket_and_large_file_v2() {
}
setup_bucket_and_large_file_v3() {
if ! check_param_count_v2 "bucket env var" 1 $#; then
if ! check_param_count_ge_le "bucket env var, file size in MBs (optional)" 1 2 $#; then
return 1
fi
if ! bucket_name=$(setup_bucket_v3 "$1" 2>&1); then
log 2 "error setting up bucket: $bucket_name"
local response bucket_name file_name error
if ! response=$(setup_bucket_v3 "$1" 2>&1); then
log 2 "error setting up bucket: $response"
return 1
fi
if ! file_name=$(create_large_file "$file_name" 2>&1); then
log 2 "error creating large file: $file_name"
bucket_name="$response"
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! error=$(create_large_file "$file_name" "$2" 2>&1); then
log 2 "error creating large file: $error"
return 1
fi
echo "$bucket_name $file_name"
@@ -521,57 +531,139 @@ compare_files() {
return 1
}
# Usage: create_large_file [filename] [size_in_mb]
# If filename is omitted, it generates one. Defaults to 160MB.
# generate 160MB file
# input: filename
# fail on error
create_large_file() {
if ! check_param_count_le "filename (optional), size in MB (optional)" 2 $#; then
log 6 "create_large_file"
if ! check_param_count_ge_le "file name, size in MBs (optional)" 1 2 $#; then
return 1
fi
local file_name="$1"
local size_mb="${2:-160}"
local error
if [ -z "$TEST_FILE_FOLDER" ]; then
log 2 "TEST_FILE_FOLDER must be defined"
file_size=${2:-160}
if ! error=$(create_large_file_with_size "$1" "$file_size" 2>&1); then
log 2 "error creating 160MB file: $error"
return 1
fi
return 0
}
if [[ -z "$file_name" ]]; then
if ! file_name=$(get_file_name 2>&1); then
log 2 "error generating automatic file name: $file_name"
create_large_file_v2() {
local response file_name
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! error=$(create_large_file_with_size "$file_name" 160 2>&1); then
log 2 "error creating 160MB file with name '$file_name': $error"
return 1
fi
echo "$file_name"
return 0
}
create_large_file_with_size() {
if ! check_param_count_v2 "file name, size in MB" 2 $#; then
return 1
fi
file_size=$(($2*1024*1024))
if ! error=$(create_test_file_base "$1" "$file_size" 2>&1); then
log 2 "error creating large file with size ${2}MB: $error"
return 1
fi
return 0
}
create_test_file_base() {
if ! check_param_count_ge_le "file name, size (optional), block size (optional)" 1 3 $#; then
return 1
fi
log 5 "params: $*"
local file_name=$1
local requested_size=${2:-10}
local block_size=${3:-1048576}
if [ "$file_name" == "" ]; then
log 2 "no file name specified"
return 1
fi
local full_path="$TEST_FILE_FOLDER/$file_name"
if [[ "$requested_size" -eq 0 ]]; then
touch "$full_path"
elif [[ "$requested_size" -lt "$block_size" ]]; then
# For small files, use the size as the block size (1 write)
if ! error=$(dd if=/dev/urandom of="$full_path" bs="$requested_size" count=1 conv=notrunc 2>&1); then
log 2 "error writing file: $error"
return 1
fi
else
# For large files, use 1MB chunks to save RAM
count=$(( requested_size / block_size ))
remainder=$(( requested_size % block_size ))
# Write the big 1MB chunks
if ! error=$(dd if=/dev/urandom of="$full_path" bs="$block_size" count="$count" conv=notrunc 2>&1); then
log 2 "error writing file chunk: $error"
return 1
fi
# If there's a remainder (e.g., 1.5MB), append the last few bytes
if [[ $remainder -gt 0 ]] && ! error=$(dd if=/dev/urandom of="$full_path" bs=1 count="$remainder" oflag=append conv=notrunc 2>&1); then
log 2 "error writing final file chunk: $error"
return 1
fi
fi
log 6 "Creating ${size_mb}MB file: $file_name"
# bs=1M is significantly faster than bs=1024 for large files
if ! error=$(dd if=/dev/urandom of="${TEST_FILE_FOLDER}/${file_name}" bs=1M count="$size_mb" 2>&1); then
log 2 "error creating ${size_mb}MB file at ${file_name}: $error"
return 1
fi
echo "$file_name"
return 0
}
create_and_split_large_file() {
if ! check_param_count_v2 "file name, size in MB, pieces" 3 $#; then
return 1
fi
if ! error=$(create_large_file_with_size "$1" "$2" 2>&1); then
log 2 "error creating large file: $error"
return 1
fi
if ! split_file "$TEST_FILE_FOLDER/$1" "$3"; then
log 2 "error splitting file"
return 1
fi
}
# param: number of files
# fail on error
create_test_file_count() {
if ! check_param_count_v2 "number of files" 1 $#; then
return 1
fi
if [ -z "$TEST_FILE_FOLDER" ]; then
log 2 "TEST_FILE_FOLDER not defined"
return 1
fi
local response file_prefix
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file prefix: $response"
return 1
fi
file_prefix="$response"
for ((i=1;i<=$1;i++)) {
if ! error=$(touch "$TEST_FILE_FOLDER/file_$i" 2>&1); then
if ! error=$(touch "$TEST_FILE_FOLDER/${file_prefix}_${i}" 2>&1); then
log 2 "error creating file_$i: $error"
return 1
fi
}
# shellcheck disable=SC2153
if [[ $LOG_LEVEL -ge 5 ]]; then
ls_result=$(ls "$TEST_FILE_FOLDER/file_*")
ls_result=$(ls "$TEST_FILE_FOLDER/${file_prefix}_*" 2>&1)
log 5 "$ls_result"
fi
echo "$file_prefix"
return 0
}
@@ -579,7 +671,7 @@ download_and_compare_file_with_user() {
if ! check_param_count_gt "original file, bucket, key, destination, username, password, chunk size (optional)" 6 $#; then
return 1
fi
if [ -e "$4" ] && ! error=$(rm -f "$4"); then
if [ -e "$4" ] && ! error=$(rm -f "$4" 2>&1); then
log 2 "error deleting local file at download destination before download: $error"
return 1
fi
tests/drivers/get_object_tagging/get_object_tagging_rest.sh
+3 -3
View File
@@ -88,10 +88,10 @@ get_check_object_tags_empty() {
}
check_header_version_id() {
if ! check_param_count_v2 "data file" 1 $#; then
if ! check_param_count_v2 "data file, version ID" 2 $#; then
return 1
fi
if ! check_for_header_key_and_value "$1" "x-amz-version-id" "$version_id"; then
if ! check_for_header_key_and_value "$1" "x-amz-version-id" "$2"; then
log 2 "error checking for x-amz-version-id header"
return 1
fi
@@ -107,7 +107,7 @@ add_version_tags_check_version_id() {
return 1
fi
if ! send_rest_go_command_callback "200" "check_header_version_id" "-bucketName" "$1" "-objectKey" "$2" "-debug" "-logFile" "signature.log" \
"-method" "GET" "-query" "tagging=&versionId=$version_id" "-tagKey" "key" "-tagValue" "value" "-contentMD5"; then
"-method" "GET" "-query" "tagging=&versionId=$version_id" "-tagKey" "key" "-tagValue" "value" "-contentMD5" "--" "$version_id"; then
log 2 "error tagging object"
return 1
fi
tests/drivers/head_object/head_object_rest.sh
+16 -5
View File
@@ -60,10 +60,14 @@ check_checksum_rest() {
return 1
fi
header_key="$4"
if ! head_object_rest_expect_success_callback "$1" "$2" "CHECKSUM=true" "parse_head_checksum"; then
log 2 "error calling HeadObject command"
local response
if ! response=$(head_object_rest_expect_success_callback "$1" "$2" "CHECKSUM=true" "parse_head_checksum" 2>&1); then
log 2 "error calling HeadObject command: $response"
return 1
fi
head_checksum="$response"
if [ "$3" != "$head_checksum" ]; then
log 2 "'checksum mismatch (head '$head_checksum', local '$3')"
return 1
@@ -76,6 +80,8 @@ parse_head_checksum() {
return 1
fi
head_checksum=$(grep -i "$header_key" "$1" | awk '{print $2}' | sed 's/\r$//')
echo "$head_checksum"
return 0
}
verify_checksum_doesnt_exist() {
@@ -93,7 +99,8 @@ parse_content_length() {
if ! check_param_count_v2 "file" 1 $#; then
return 1
fi
content_length=$(grep "Content-Length:" "$1" | awk '{print $2}' | tr -d '\r')
content_length="$(grep "Content-Length:" "$1" | awk '{print $2}' | tr -d '\r')"
echo "$content_length"
return 0
}
@@ -163,10 +170,14 @@ get_object_size_with_user() {
if ! check_param_count_v2 "username, password, bucket, key" 4 $#; then
return 1
fi
if ! head_object_rest_expect_success_callback "$3" "$4" "AWS_ACCESS_KEY_ID=$1 AWS_SECRET_ACCESS_KEY=$2" "parse_content_length"; then
log 2 "error getting object size"
local response
if ! response=$(head_object_rest_expect_success_callback "$3" "$4" "AWS_ACCESS_KEY_ID=$1 AWS_SECRET_ACCESS_KEY=$2" "parse_content_length" 2>&1); then
log 2 "error getting object size: $response"
return 1
fi
content_length="$response"
log 5 "file size: $content_length"
echo "$content_length"
return 0
tests/drivers/list_buckets/list_buckets_rest.sh
+65 -20
View File
@@ -15,10 +15,10 @@
# under the License.
bucket_exists_in_list() {
if ! check_param_count_v2 "bucket" 1 $#; then
if ! check_param_count_gt "bucket, buckets in list" 2 $#; then
return 1
fi
for bucket in "${bucket_array[@]}"; do
for bucket in "${@:2}"; do
if [ "$bucket" == "$1" ]; then
return 0
fi
@@ -41,18 +41,21 @@ list_check_buckets_rest_with_params() {
if ! check_param_count_gt "params, expected buckets" 2 $#; then
return 1
fi
if ! list_buckets_rest "$1" "parse_bucket_list"; then
log 2 "error listing buckets"
local response
if ! response=$(list_buckets_rest "$1" "parse_bucket_list" 2>&1); then
log 2 "error listing buckets: $response"
return 1
fi
read -r -a bucket_array <<< "$response"
for bucket in "${@:2}"; do
log 5 "bucket: $bucket"
if ! bucket_exists_in_list "$bucket"; then
log 2 "bucket $bucket not found"
if ! bucket_exists_in_list "$bucket" "${bucket_array[@]}"; then
log 2 "bucket '$bucket' not found"
return 1
fi
done
return 0
return 0
}
parse_bucket_list() {
@@ -61,7 +64,16 @@ parse_bucket_list() {
fi
# shellcheck disable=SC2154
log 5 "bucket list: $(cat "$1")"
bucket_list=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$1")
local response
if ! response=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$1" 2>&1); then
if [[ "$response" == *"XPath set is empty"* ]]; then
return 0
fi
log 2 "error retrieving bucket list: $response"
return 1
else
bucket_list="$response"
fi
bucket_array=()
while read -r bucket; do
if [ -n "$bucket" ]; then
@@ -69,28 +81,50 @@ parse_bucket_list() {
bucket_array+=("$bucket")
fi
done <<< "$bucket_list"
log 5 "bucket array: ${bucket_array[*]}"
log 5 "bucket array length: ${#bucket_array[@]}"
# return val
echo "${bucket_array[*]}"
return 0
}
parse_buckets_and_continuation_token() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
if ! parse_bucket_list "$1"; then
log 2 "error parsing bucket list"
local response buckets continuation_token
if ! response=$(parse_bucket_list "$1" 2>&1); then
log 2 "error parsing bucket list: $response"
return 1
fi
continuation_token=$(xmllint --xpath '//*[local-name()="ListAllMyBucketsResult"]/*[local-name()="ContinuationToken"]/text()' "$1")
log 5 "token: $continuation_token"
buckets="$response"
if ! response=$(xmllint --xpath '//*[local-name()="ListAllMyBucketsResult"]/*[local-name()="ContinuationToken"]/text()' "$1" 2>&1); then
if [[ "$response" == *"XPath set is empty"* ]]; then
continuation_token=
else
log 2 "error getting continuation token: $response"
return 1
fi
else
continuation_token="$response"
fi
# return vals
echo "$continuation_token"
echo "$buckets"
return 0
}
check_continuation_token() {
if ! list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token"; then
log 2 "error listing buckets"
local response
if ! response=$(list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token" 2>&1); then
log 2 "error listing buckets: $response"
return 1
fi
mapfile -t continuation_token_buckets_lines <<< "$response"
continuation_token="${continuation_token_buckets_lines[0]}"
read -r -a bucket_array <<< "${continuation_token_buckets_lines[1]}"
if [ ${#bucket_array[@]} != "1" ]; then
log 2 "expected one bucket to be returned, was ${#bucket_array}"
return 1
@@ -105,10 +139,16 @@ check_for_buckets_with_multiple_pages() {
if ! check_param_count_v2 "buckets" 2 $#; then
return 1
fi
if ! list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token"; then
log 2 "error listing buckets"
local response
if ! response=$(list_buckets_rest "MAX_BUCKETS=1" "parse_buckets_and_continuation_token" 2>&1); then
log 2 "error listing buckets: $response"
return 1
fi
mapfile -t continuation_token_buckets_lines <<< "$response"
continuation_token="${continuation_token_buckets_lines[0]}"
read -r -a bucket_array <<< "${continuation_token_buckets_lines[1]}"
bucket_one_found="false"
bucket_two_found="false"
while true; do
@@ -123,10 +163,13 @@ check_for_buckets_with_multiple_pages() {
if [ "$continuation_token" == "" ]; then
break
fi
if ! list_buckets_rest "MAX_BUCKETS=1 CONTINUATION_TOKEN=$continuation_token" "parse_buckets_and_continuation_token"; then
log 2 "error"
if ! response=$(list_buckets_rest "MAX_BUCKETS=1 CONTINUATION_TOKEN=$continuation_token" "parse_buckets_and_continuation_token" 2>&1); then
log 2 "error listing buckets: $response"
return 1
fi
mapfile -t continuation_token_buckets_lines <<< "$response"
continuation_token="${continuation_token_buckets_lines[0]}"
read -r -a bucket_array <<< "${continuation_token_buckets_lines[1]}"
done
if [ "$bucket_one_found" == "false" ]; then
log 2 "bucket '$1' not found in list"
@@ -313,3 +356,5 @@ list_check_bucket_and_region() {
fi
return 0
}
tests/drivers/list_object_versions/list_object_versions_rest.sh
+50 -24
View File
@@ -35,6 +35,7 @@ parse_version_or_delete_marker_id() {
return 1
fi
log 5 "version or marker ID: $version_or_marker_id"
echo "$version_or_marker_id"
return 0
}
@@ -42,33 +43,43 @@ parse_version_id() {
if ! check_param_count_v2 "data file, IsLatest val" 2 $#; then
return 1
fi
if ! parse_version_or_delete_marker_id "$1" "Version" "$2"; then
echo "error parsing version ID"
if ! version_id=$(parse_version_or_delete_marker_id "$1" "Version" "$2" 2>&1); then
echo "error parsing version ID: $version_id"
return 1
fi
version_id=$version_or_marker_id
echo "$version_id"
return 0
}
parse_non_latest_version_id() {
parse_a_non_latest_version_id() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
if ! parse_version_id "$1" "false"; then
log 2 "error getting non-latest version ID"
local response
if ! response=$(parse_version_id "$1" "false" 2>&1); then
log 2 "error getting non-latest version ID: $response"
return 1
fi
version_id="$response"
echo "$version_id"
return 0
}
get_non_latest_version() {
get_a_non_latest_version() {
if ! check_param_count_v2 "bucket" $# 1; then
return 1
fi
if ! send_rest_go_command_callback "200" "parse_non_latest_version_id" "-method" "GET" "-query" "versions=" "-bucketName" "$1"; then
log 2 "error retrieving version tags"
local response
if ! response=$(send_rest_go_command_callback "200" "parse_a_non_latest_version_id" "-method" "GET" "-query" "versions=" "-bucketName" "$1" 2>&1); then
log 2 "error retrieving non-latest version ID: $response"
return 1
fi
non_latest_version="$response"
echo "$non_latest_version"
return 0
}
@@ -76,11 +87,15 @@ check_object_versions_before_deletion() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
if ! get_xml_data "$1" "$1.tmp"; then
log 2 "error getting XML data"
local response
if ! response=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then
log 2 "error getting XML data: $response"
return 1
fi
if ! parse_versions_rest "$1.tmp"; then
xml_data="$response"
if ! parse_versions_rest "$xml_data"; then
log 2 "error parsing versions"
return 1
fi
@@ -90,18 +105,23 @@ check_object_versions_before_deletion() {
fi
version_id="${version_ids[0]}"
log 5 "version ID: $version_id"
echo "$version_id"
return 0
}
check_object_versions_after_deletion() {
if ! check_param_count_v2 "data file" 1 $#; then
if ! check_param_count_v2 "data file, version ID" 2 $#; then
return 1
fi
if ! get_xml_data "$1" "$1.tmp"; then
log 2 "error getting XML data"
local response
if ! response=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then
log 2 "error getting XML data: $response"
return 1
fi
if ! parse_versions_rest "$1.tmp"; then
xml_data="$response"
if ! parse_versions_rest "$xml_data"; then
log 2 "error parsing versions"
return 1
fi
@@ -109,8 +129,8 @@ check_object_versions_after_deletion() {
log 2 "expected version ID count of 2, was '${#version_ids[@]}'"
return 1
fi
if [ "${version_ids[0]}" != "$version_id" ]; then
log 2 "expected version ID of '$version_id', was '${version_ids[0]}'"
if [ "${version_ids[0]}" != "$2" ]; then
log 2 "expected version ID of '$2', was '${version_ids[0]}'"
return 1
fi
if [ "${version_islatests[0]}" != "false" ]; then
@@ -128,8 +148,8 @@ list_object_versions_before_and_after_retention_deletion() {
if ! check_param_count_v2 "bucket name, file" 2 $#; then
return 1
fi
if ! send_rest_go_command_callback "200" "check_object_versions_before_deletion" \
"-method" "GET" "-bucketName" "$1" "-query" "versions="; then
if ! key_version_id=$(send_rest_go_command_callback "200" "check_object_versions_before_deletion" \
"-method" "GET" "-bucketName" "$1" "-query" "versions=" 2>&1); then
log 2 "error checking versions before deletion"
return 1
fi
@@ -137,9 +157,10 @@ list_object_versions_before_and_after_retention_deletion() {
log 2 "error deleting file"
return 1
fi
log 5 "version ID: $key_version_id"
if ! send_rest_go_command_callback "200" "check_object_versions_after_deletion" \
"-method" "GET" "-bucketName" "$1" "-query" "versions="; then
log 2 "error checking versions before deletion"
"-method" "GET" "-bucketName" "$1" "-query" "versions=" "--" "$key_version_id"; then
log 2 "error checking versions after deletion"
return 1
fi
return 0
@@ -149,10 +170,15 @@ parse_latest_version_id() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
if ! parse_version_id "$1" "true"; then
log 2 "error getting latest version ID"
local response
if ! response=$(parse_version_id "$1" "true" 2>&1); then
log 2 "error getting latest version ID: $response"
return 1
fi
version_id="$response"
log 5 "version ID: $version_id"
echo "$version_id"
return 0
}
tests/drivers/list_objects/list_objects_rest.sh
+175 -4
View File
@@ -18,10 +18,13 @@ list_check_single_object() {
if ! check_param_count_gt "bucket, key, env params (optional)" 2 $#; then
return 1
fi
if ! list_objects_rest "$1" "parse_objects_list_rest" "$3"; then
log 2 "error listing objects"
local response
if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" "$3" 2>&1); then
log 2 "error listing objects: $response"
return 1
fi
mapfile -t object_array <<< "$response"
if [ ${#object_array[@]} -ne "1" ]; then
log 2 "expected one object, found ${#object_array[@]}"
return 1
@@ -154,10 +157,10 @@ check_prefix_delimiter_and_counts() {
}
list_objects_with_prefix_and_delimiter_check_results() {
if ! check_param_count_gt "bucket name, prefix, delimiter, expected common prefixes, --, expected keys" 5 $#; then
if ! check_param_count_gt "bucket name, ListObjects version, prefix, delimiter, expected common prefixes, --, expected keys" 6 $#; then
return 1
fi
if ! send_rest_go_command_callback "200" "check_common_prefixes_and_keys" "-bucketName" "$1" "-query" "delimiter=$3&prefix=$2" "--" "${@:2}"; then
if ! send_rest_go_command_callback "200" "check_common_prefixes_and_keys" "-bucketName" "$1" "-query" "list-type=$2&delimiter=$4&prefix=$3" "--" "${@:3}"; then
log 2 "error sending command to list objects or receiving response"
return 1
fi
@@ -208,6 +211,7 @@ check_count_and_keys() {
return 1
fi
done
echo "$xml_data"
return 0
}
@@ -223,3 +227,170 @@ list_objects_check_count_and_keys() {
fi
return 0
}
check_count_keys_and_get_token() {
if ! check_param_count_gt "data file, expected continuation token, count, last keys, keys" 4 $#; then
return 1
fi
local response error
if ! response=$(check_count_and_keys "$1" "$3" "${@:5}" 2>&1); then
log 2 "error checking count and keys: $response"
return 1
fi
xml_file="$response"
if [ "$2" != "" ] && ! error=$(check_if_element_exists "$xml_file" "$2" "ListBucketResult" "ContinuationToken" 2>&1); then
log 2 "error getting continuation token: $error"
return 1
fi
if [ "$4" == "false" ]; then
if ! response=$(get_element_text "$xml_file" "ListBucketResult" "NextContinuationToken" 2>&1); then
log 2 "error getting next continuation token: $response"
return 1
fi
next_continuation_token="$response"
else
if response=$(get_element_text "$xml_file" "ListBucketResult" "NextContinuationToken" 2>/dev/null) && [ -n "$response" ]; then
log 2 "last element shouldn't have 'NextConfigurationToken' value"
return 1
fi
next_continuation_token=
fi
echo "$next_continuation_token"
return 0
}
list_objects_v2_check_count_and_keys_get_token() {
if ! check_param_count_gt "bucket name, expected token, count, last keys, keys, additional params if any" 4 $#; then
return 1
fi
local count="$3"
local keys=("${@:5:$count}")
if ! callback_response=$(send_rest_go_command_callback "200" "check_count_keys_and_get_token" "-bucketName" "$1" "${@:((5+$count))}" "--" "$2" "$count" "$4" "${keys[@]}" 2>&1); then
log 2 "error sending list objects command: $callback_response"
return 1
else
continuation_token="$callback_response"
fi
echo "$continuation_token"
return 0
}
check_start_after_no_continuation_token() {
if ! check_param_count_v2 "bucket, last file in alphabetical order" 2 $#; then
return 1
fi
if ! send_rest_go_command_callback "200" "verify_element_doesnt_exist" "-bucketName" "$1" "-query" "start-after=$2&list-type=2" "--" "$2" "ListBucketResult" "ContinuationToken"; then
log 2 "error verifying that ContinuationToken value is not returned"
return 1
fi
return 0
}
list_objects_check_start_after_response() {
if ! check_param_count_gt "bucket, start after token, listed files" 2 $#; then
return 1
fi
if ! send_rest_go_command_callback "200" "check_start_after_response" "-bucketName" "$1" "-query" "start-after=$2&list-type=2" "--" "$2" "${@:3}"; then
log 2 "error listing objects and checking start-after response"
return 1
fi
return 0
}
check_start_after_response() {
if ! check_param_count_gt "data file, start after file, listed files" 2 $#; then
return 1
fi
local response
if ! response=$(check_validity_and_or_parse_xml_data "$1" 2>&1); then
log 2 "error parsing xml data: $response"
return 1
fi
xml_data="$response"
if ! check_if_element_exists "$xml_data" "$2" "ListBucketResult" "StartAfter"; then
log 2 "error checking if element '$2' exists"
return 1
fi
local count
if [ "$3" == "" ]; then
count=0
else
count=$(($#-2))
fi
if ! check_element_count "$xml_data" "$count" "ListBucketResult" "Contents" "Key"; then
log 2 "error checking element count"
return 1
fi
for file_name in "${@:3}"; do
if ! check_if_element_exists "$xml_data" "$file_name" "ListBucketResult" "Contents" "Key"; then
log 2 "error checking if element '$file_name' exists"
return 1
fi
done
return 0
}
list_objects_verify_owner_info_missing() {
if ! check_param_count_gt "bucket name, files" 1 $#; then
return 1
fi
if ! send_rest_go_command_callback "200" "verify_owner_info_missing" "-bucketName" "$1" "-query" "list-type=2&fetch-owner=false" "--" "${@:2}"; then
log 2 "error sending list objects v2 command and verifying that the owner data is missing"
return 1
fi
}
verify_owner_info_missing() {
if ! check_param_count_gt "data file, files" 1 $#; then
return 1
fi
local response
for key in "${@:2}"; do
if ! response=$(get_element_with_matching_inner_value "$1" "$key" "ListBucketResult" "Contents" "--" "Key" 2>&1); then
log 2 "error finding element matching key '$key': $response"
return 1
fi
element="$response"
if check_xml_element_inside_string "$element" "$AWS_ACCESS_KEY_ID" "Owner" "ID"; then
log 2 "'Owner' value should not be present"
return 1
fi
done
return 0
}
list_objects_verify_owner_info_exists() {
if ! check_param_count_gt "bucket name, files" 1 $#; then
return 1
fi
if ! send_rest_go_command_callback "200" "verify_owner_info_exists" "-bucketName" "$1" "-query" "list-type=2&fetch-owner=true" "--" "${@:2}"; then
log 2 "error sending list objects v2 command and verifying that the owner data is missing"
return 1
fi
}
verify_owner_info_exists() {
if ! check_param_count_gt "data file, files" 1 $#; then
return 1
fi
local response
for key in "${@:2}"; do
if ! response=$(get_element_with_matching_inner_value "$1" "$key" "ListBucketResult" "Contents" "--" "Key" 2>&1); then
log 2 "error finding element matching key '$key': $response"
else
element="$response"
fi
if ! check_xml_element_inside_string "$element" "$AWS_ACCESS_KEY_ID" "Owner" "ID"; then
log 2 "'Owner' value missing"
return 1
fi
done
return 0
}
tests/drivers/params.sh
+12
View File
@@ -73,3 +73,15 @@ check_param_count_le() {
fi
return 0
}
check_param_count_ge_le() {
if [ $# -ne 4 ]; then
log 2 "'check_param_count_ge_le' requires params list, expected minimum, expected maximum, actual"
return 1
fi
if [ "$2" -gt "$4" ] || [ "$3" -lt "$4" ]; then
log_with_stack_ref 2 "function '${FUNCNAME[1]}' requires $1" 2
return 1
fi
return 0
}
tests/drivers/put_object_tagging/put_object_tagging_rest.sh
+6 -2
View File
@@ -44,10 +44,14 @@ tag_old_version() {
if ! check_param_count_v2 "bucket name, key" 2 $#; then
return 1
fi
if ! get_non_latest_version "$1"; then
log 2 "error getting non-latest object version"
local response
if ! response=$(get_a_non_latest_version "$1" 2>&1); then
log 2 "error getting non-latest object version: $response"
return 1
fi
version_id="$response"
if ! send_rest_go_command "200" "-bucketName" "$1" "-objectKey" "$2" "-query" "versionId=$version_id" "-debug" "-logFile" "signature.log" \
"-commandType" "putObjectTagging" "-tagKey" "key" "-tagValue" "value" "-contentMD5"; then
log 2 "error tagging object"
tests/drivers/rest.sh
+56 -24
View File
@@ -36,12 +36,15 @@ check_rest_go_expected_error() {
return 1
fi
result="$(cat "$1")"
if ! bypass_continues; then
log 2 "error bypassing continues"
local response
if ! response=$(bypass_continues "$result" 2>&1); then
log 2 "error bypassing continues: $response"
return 1
fi
status_code=$(echo -n "$response" | awk '{print $2}')
if [ "$2" != "$status_code" ]; then
log 2 "expected curl response '$2', was '$status_code'"
log 2 "expected curl response '$2', was '$status_code' (response: '$result')"
return 1
fi
if ! check_xml_error_contains "$1" "$3" "$4"; then
@@ -99,13 +102,14 @@ check_rest_expected_header_error() {
return 1
fi
result="$(cat "$1")"
if ! bypass_continues; then
log 2 "error bypassing continues"
local response
if ! response=$(bypass_continues "$result" 2>&1); then
log 2 "error bypassing continues: $response"
return 1
fi
log 5 "status line: $status_line"
status_message=$(echo "$status_line" | cut -d' ' -f3- | tr -d '\r')
log 5 "status code: $status_code, status message: $status_message"
status_code=$(echo "$response" | awk '{print $2}')
status_message=$(echo "$response" | cut -d' ' -f3- | tr -d '\r')
if [ "$2" != "$status_code" ]; then
log 2 "expected curl response '$2', was '$status_code' ($(echo -n "$result"))"
return 1
@@ -171,10 +175,15 @@ send_rest_command_expect_success_callback() {
log 2 "expected '$3', was '$response_code' ($(cat "$output_file"))"
return 1
fi
if [ "$4" != "" ] && ! "$4" "$output_file"; then
log 2 "callback error"
if [ "$4" == "" ]; then
cat "$output_file"
return 0
fi
if ! callback_result=$("$4" "$output_file" 2>&1); then
log 2 "callback error: $callback_result"
return 1
fi
echo "$callback_result"
return 0
}
@@ -196,6 +205,7 @@ rest_go_command_perform_send() {
return 1
fi
echo "$result"
return 0
}
send_rest_go_command_expect_error() {
@@ -250,11 +260,13 @@ send_rest_go_command_expect_error_callback() {
}
bypass_continues() {
status_line_idx=1
status_code=""
continue_count=0
if ! check_param_count_v2 "raw response" 1 $#; then
return 1
fi
local status_line_idx=1 status_code="" continue_count=0
while ((continue_count<10)); do
status_line=$(sed -n "${status_line_idx}p" <<< "$result")
status_line=$(sed -n "${status_line_idx}p" <<< "$1")
status_code=$(echo "$status_line" | awk '{print $2}')
if [ "$status_code" != "100" ]; then
break
@@ -266,6 +278,7 @@ bypass_continues() {
log 2 "too many continues"
return 1
fi
echo "$status_line"
return 0
}
@@ -311,7 +324,7 @@ send_rest_go_command_callback() {
return 1
fi
local all_params=("${@:3}") no_callback_params=0 go_param_array=() callback_params=()
local all_params=("${@:3}") no_callback_params=0 go_param_array=() callback_params=() response
if ! params_file=$(get_file_name 2>&1); then
log 2 "error getting params file name: $params_file"
@@ -324,27 +337,46 @@ send_rest_go_command_callback() {
mapfile -t callback_params < <(get_callback_params "${all_params[@]}")
fi
if ! rest_go_command_perform_send "${go_param_array[@]}"; then
log 2 "error sending rest go command"
if ! result=$(rest_go_command_perform_send "${go_param_array[@]}" 2>&1); then
log 2 "error sending rest go command: $result"
return 1
fi
if ! bypass_continues; then
log 2 "error bypassing continues"
if ! response=$(bypass_continues "$result" 2>&1); then
log 2 "error bypassing continues: $response"
return 1
fi
status_code=$(echo -n "$response" | awk '{print $2}')
if [ "$1" != "$status_code" ]; then
log 2 "expected curl response '$1', was '$status_code'"
log 2 "expected curl response '$1', was '$status_code' (response: '$result')"
return 1
fi
if ! output_file_name=$(get_file_name); then
if [ "$2" == "" ]; then
echo "$result"
return 0
fi
if ! callback_result=$(call_callback "$result" "$2" 2>&1); then
log 2 "callback error: $callback_result"
return 1
fi
echo "$callback_result"
return 0
}
call_callback() {
if ! check_param_count_v2 "response, callback" 2 $#; then
return 1
fi
if ! output_file_name=$(get_file_name 2>&1); then
log 2 "error generating output file name: $output_file_name"
return 1
fi
echo -n "$result" > "$TEST_FILE_FOLDER/$output_file_name"
if [ "$2" != "" ] && ! "$2" "$TEST_FILE_FOLDER/$output_file_name" "${callback_params[@]}"; then
log 2 "error in callback"
echo -n "$1" > "$TEST_FILE_FOLDER/$output_file_name"
local callback_output
if ! callback_output=$("$2" "$TEST_FILE_FOLDER/$output_file_name" "${callback_params[@]}" 2>&1); then
log 2 "error in callback: $callback_output"
return 1
fi
echo "$callback_output"
return 0
}
tests/drivers/upload_part/upload_part_rest.sh
+18 -6
View File
@@ -18,14 +18,22 @@ multipart_upload_rest_before_completion() {
if ! check_param_count_v2 "bucket, key, file, part count" 4 $#; then
return 1
fi
if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then
log 2 "error creating multipart upload"
local response
if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then
log 2 "error creating multipart upload: $response"
return 1
fi
if ! upload_parts_rest_before_completion "$1" "$2" "$3" "$upload_id" "$4"; then
log 2 "error uploading parts before completion"
upload_id="$response"
if ! response=$(upload_parts_rest_before_completion "$1" "$2" "$3" "$upload_id" "$4" 2>&1); then
log 2 "error uploading parts before completion: $response"
return 1
fi
parts_payload="$response"
echo "$upload_id $parts_payload"
return 0
}
upload_parts_rest_before_completion() {
@@ -92,10 +100,14 @@ perform_full_multipart_upload_with_checksum_before_completion() {
log 2 "error setting up bucket and large file"
return 1
fi
if ! create_multipart_upload_rest "$bucket_name" "$2" "CHECKSUM_TYPE=$3 CHECKSUM_ALGORITHM=$4" "parse_upload_id"; then
log 2 "error creating multipart upload"
local response
if ! response=$(create_multipart_upload_rest "$bucket_name" "$2" "CHECKSUM_TYPE=$3 CHECKSUM_ALGORITHM=$4" "parse_upload_id" 2>&1); then
log 2 "error creating multipart upload: $response"
return 1
fi
upload_id="$response"
lowercase_checksum_algorithm=$(echo -n "$4" | tr '[:upper:]' '[:lower:]')
if ! upload_parts_rest_with_checksum_before_completion "$bucket_name" "$2" "$TEST_FILE_FOLDER/$2" "$upload_id" 2 "$lowercase_checksum_algorithm"; then
log 2 "error uploading parts"
tests/drivers/xml.sh
+15 -1
View File
@@ -246,7 +246,9 @@ check_if_element_exists() {
log 2 "error checking XML data: $data_file"
return 1
fi
if ! result=$(xmllint --xpath "boolean(${xpath}[text()='$2'])" "$data_file" 2>&1); then
local search_string="boolean(${xpath}[text()='$2'])"
log 5 "search string: $search_string"
if ! result=$(xmllint --xpath "$search_string" "$data_file" 2>&1); then
log 2 "error getting result: $result"
return 1
fi
@@ -257,6 +259,18 @@ check_if_element_exists() {
return 1
}
verify_element_doesnt_exist() {
if ! check_param_count_gt "data file, element, xml tree" 3 $#; then
return 1
fi
if check_if_element_exists "$1" "$2" "${@:3}"; then
log 2 "element '$2' should not exist in data"
log 5 "data: $(cat "$1")"
return 1
fi
return 0
}
print_xml_data_to_file() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
tests/rest_scripts/command/canonicalQuery.go
+23 -18
View File
@@ -34,12 +34,31 @@ func canonicalizeQuery(raw string) (string, error) {
raw += "="
}
vals, err := url.ParseQuery(raw)
if err != nil {
return "", fmt.Errorf("error parsing query: %w", err)
// Parse manually instead of url.ParseQuery so we don't treat '+' as space.
// S3 continuation tokens are opaque and may contain literal '+' characters.
pairs := make([]queryPair, 0)
for _, part := range strings.Split(raw, "&") {
if part == "" {
continue
}
kv := strings.SplitN(part, "=", 2)
keyRaw := kv[0]
valRaw := ""
if len(kv) == 2 {
valRaw = kv[1]
}
key, err := url.PathUnescape(keyRaw)
if err != nil {
return "", fmt.Errorf("error unescaping query key '%s': %w", keyRaw, err)
}
val, err := url.PathUnescape(valRaw)
if err != nil {
return "", fmt.Errorf("error unescaping query value for key '%s': %w", keyRaw, err)
}
pairs = append(pairs, queryPair{key: key, value: val})
}
pairs := getQueryPairs(vals)
sort.Slice(pairs, func(i, j int) bool {
escapedKeyI, escapedKeyJ := awsQueryEscape(pairs[i].key), awsQueryEscape(pairs[j].key)
if escapedKeyI != escapedKeyJ {
@@ -60,17 +79,3 @@ func canonicalizeQuery(raw string) (string, error) {
}
return b.String(), nil
}
func getQueryPairs(values url.Values) []queryPair {
pairs := make([]queryPair, 0, len(values))
for queryKey, queryValues := range values {
if len(queryValues) == 0 {
pairs = append(pairs, queryPair{key: queryKey, value: ""})
continue
}
for _, v := range queryValues {
pairs = append(pairs, queryPair{key: queryKey, value: v})
}
}
return pairs
}
tests/test_common.sh
+2 -1
View File
@@ -331,12 +331,13 @@ test_common_list_objects_file_count() {
run create_test_file_count 1001
assert_success
prefix=$output
run setup_bucket_v3 "$BUCKET_ONE_NAME"
assert_success
bucket_name=$output
run put_object_multiple "$1" "$TEST_FILE_FOLDER/file_*" "$bucket_name"
run put_object_multiple "$1" "$TEST_FILE_FOLDER/${prefix}_*" "$bucket_name"
assert_success
run list_objects_check_file_count "$1" "$bucket_name" 1001
tests/test_rest_list_objects.sh
+124 -19
View File
@@ -155,25 +155,7 @@ source ./tests/drivers/list_objects/list_objects_rest.sh
}
@test "REST - ListObjects - delimiter" {
run get_bucket_name "$BUCKET_ONE_NAME"
assert_success
local bucket_name="$output"
file_names=("a-b-1.txt" "a-b-2.txt" "a-b/c-1.txt" "a-b/c-2.txt" "a-b/d.txt" "a/c.txt")
local prefix="a-"
run create_test_files_and_folders "${file_names[@]}"
assert_success
run setup_bucket_v2 "$bucket_name"
assert_success
for file_name in "${file_names[@]}"; do
run put_object "rest" "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
assert_success
done
run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt"
assert_success
list_objects_delimiter "1"
}
@test "REST - ListObjects - invalid encoding" {
@@ -262,3 +244,126 @@ source ./tests/drivers/list_objects/list_objects_rest.sh
run list_objects_check_count_and_keys "$bucket_name" "1" "$file_two_name" "-query" "prefix=$file_two_name"
assert_success
}
@test "REST - ListObjectsV2 - continuation token" {
run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 3
assert_success
read -r bucket_name file_one file_two file_three <<< "$output"
mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" "$file_three" | sort)
run put_objects "$bucket_name" "$file_one" "$file_two" "$file_three"
assert_success
run list_objects_v2_check_count_and_keys_get_token "$bucket_name" "" "1" "false" "${sorted_files[0]}" "-query" "list-type=2&max-keys=1"
assert_success
continuation_token=$output
log 5 "continuation token: $continuation_token"
run list_objects_v2_check_count_and_keys_get_token "$bucket_name" "$continuation_token" "1" "false" "${sorted_files[1]}" "-query" "list-type=2&max-keys=1&continuation-token=$continuation_token"
assert_success
continuation_token=$output
run list_objects_v2_check_count_and_keys_get_token "$bucket_name" "$continuation_token" "1" "true" "${sorted_files[2]}" "-query" "list-type=2&max-keys=1&continuation-token=$continuation_token"
assert_success
}
@test "ListObjectsV1 - start-after - error" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/2004"
fi
run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2
assert_success
read -r bucket_name file_one file_two <<< "$output"
mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort)
run put_objects "$bucket_name" "$file_one" "$file_two"
assert_success
run send_rest_go_command_expect_error "400" "InvalidArgument" "startAfter only supported in REST.GET.BUCKET with list-type=2" \
"-bucketName" "$bucket_name" "-query" "start-after=$file_one"
assert_success
}
@test "ListObjectsV1 - start-after - doesn't include continuation token" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/2007"
fi
run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2
assert_success
read -r bucket_name file_one file_two <<< "$output"
mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort)
run put_objects "$bucket_name" "$file_one" "$file_two"
assert_success
run check_start_after_no_continuation_token "$bucket_name" "$file_two"
assert_success
}
@test "ListObjectsV2 - start-after - success" {
run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2
assert_success
read -r bucket_name file_one file_two <<< "$output"
mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort)
run put_objects "$bucket_name" "$file_one" "$file_two"
assert_success
run list_objects_check_start_after_response "$bucket_name" "${sorted_files[0]}" "${sorted_files[1]}"
assert_success
run list_objects_check_start_after_response "$bucket_name" "${sorted_files[1]}"
assert_success
}
@test "ListObjectsV2 - fetch-owner" {
run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2
assert_success
read -r bucket_name file_one file_two <<< "$output"
mapfile -t sorted_files < <(printf '%s\n' "$file_one" "$file_two" | sort)
run put_objects "$bucket_name" "$file_one" "$file_two"
assert_success
run list_objects_verify_owner_info_missing "$bucket_name" "$file_one" "$file_two"
assert_success
run list_objects_verify_owner_info_exists "$bucket_name" "$file_one" "$file_two"
assert_success
}
# shellcheck disable=SC2030
@test "ListObjectsV2 - delimiter" {
list_objects_delimiter 2
}
list_objects_delimiter() {
run assert_param_count "ListObjects version" 1 $#
assert_success
run get_bucket_name "$BUCKET_ONE_NAME"
assert_success
# shellcheck disable=SC2031
local bucket_name="$output"
file_names=("a-b-1.txt" "a-b-2.txt" "a-b/c-1.txt" "a-b/c-2.txt" "a-b/d.txt" "a/c.txt")
local prefix="a-"
run create_test_files_and_folders "${file_names[@]}"
assert_success
run setup_bucket_v2 "$bucket_name"
assert_success
for file_name in "${file_names[@]}"; do
run put_object "rest" "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
assert_success
done
run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "2" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt"
assert_success
}
tests/test_s3api_multipart.sh
+18 -24
View File
@@ -48,17 +48,14 @@ export RUN_USERS=true
# complete-multipart-upload
@test "test_complete_multipart_upload" {
local bucket_file="bucket-file"
run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=20M count=1
run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 20
assert_success
read -r bucket_name bucket_file <<< "$output"
run multipart_upload "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
assert_success
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
assert_success
}
@@ -107,35 +104,32 @@ export RUN_USERS=true
}
@test "test-multipart-upload-from-bucket" {
local bucket_file="bucket-file"
run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=20M count=1
run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 20
assert_success
read -r bucket_name bucket_file <<< "$output"
run multipart_upload_from_bucket "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
assert_success
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "${bucket_file}-copy" "$TEST_FILE_FOLDER/$bucket_file-copy-two"
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "${bucket_file}-copy" "$TEST_FILE_FOLDER/$bucket_file-copy-two"
assert_success
}
@test "test_multipart_upload_from_bucket_range_too_large" {
local bucket_file="bucket-file"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$bucket_file"
run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 20
assert_success
read -r bucket_name bucket_file <<< "$output"
run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file"
run multipart_upload_range_too_large "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file"
assert_success
}
@test "test_multipart_upload_from_bucket_range_valid" {
local bucket_file="bucket-file"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$bucket_file"
run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 20
assert_success
read -r bucket_name bucket_file <<< "$output"
run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file"
run run_and_verify_multipart_upload_with_valid_range "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file"
assert_success
}
tests/util/util_list_objects.sh
+17 -8
View File
@@ -21,21 +21,23 @@ parse_objects_list_rest() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
object_array=()
# shellcheck disable=SC2154
if ! object_list=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then
if [[ "$object_list" == *"XPath set is empty"* ]]; then
local response
if ! response=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$1" 2>&1); then
if [[ "$response" == *"XPath set is empty"* ]]; then
return 0
fi
log 2 "error getting object list: $object_list"
log 2 "error getting object list: $response"
return 1
else
object_list="$response"
fi
log 5 "object list: '$object_list'"
while IFS= read -r object; do
log 5 "parsed key: '$object'"
object_array+=("$(echo -n "$object" | xmlstarlet unesc)")
unescaped_object="$(echo -n "$object" | xmlstarlet unesc)"
echo "$unescaped_object"
done <<< "$object_list"
log 5 "object array: ${object_array[*]}"
return 0
}
@@ -112,7 +114,14 @@ list_check_objects_rest() {
if ! check_param_count "list_check_objects_rest" "bucket" 1 $#; then
return 1
fi
list_objects_rest "$1" "parse_objects_list_rest"
local response
if ! response=$(list_objects_rest "$1" "parse_objects_list_rest" 2>&1); then
log 2 "error listing and parsing objects: $response"
return 1
fi
mapfile -t object_array <<< "$response"
object_found=false
# shellcheck disable=SC2154
for object in "${object_array[@]}"; do
tests/util/util_multipart.sh
+15 -6
View File
@@ -62,10 +62,12 @@ multipart_upload_from_bucket() {
fi
}
if ! create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id"; then
log 2 "error running first multipart upload"
local response
if ! response=$(create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id" 2>&1); then
log 2 "error running first multipart upload: $response"
return 1
fi
upload_id="$response"
if ! multipart_upload_s3api_complete_from_bucket "$1" "$2" "$4"; then
log 2 " error completing multipart upload from bucket"
@@ -112,10 +114,13 @@ multipart_upload_from_bucket_range() {
fi
}
if ! create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id"; then
log 2 "error running first multpart upload"
local response
if ! response=$(create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id" 2>&1); then
log 2 "error running first multpart upload: $response"
return 1
fi
upload_id="$response"
parts="["
for ((i = 1; i <= $4; i++)); do
if ! upload_part_copy_with_range "$1" "$2-copy" "$upload_id" "$2" "$i" "$5"; then
@@ -227,10 +232,14 @@ create_upload_part_copy_rest() {
log 2 "error splitting and putting file"
return 1
fi
if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then
log 2 "error creating upload and getting ID"
local response
if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then
log 2 "error creating upload and getting ID: $response"
return 1
fi
upload_id="$response"
parts_payload=""
for ((i=0; i<=3; i++)); do
part_number=$((i+1))
tests/util/util_multipart_abort.sh
+5 -2
View File
@@ -43,10 +43,13 @@ create_abort_multipart_upload_rest() {
return 1
fi
log 5 "uploads before upload: $(cat "$TEST_FILE_FOLDER/uploads.txt")"
if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then
log 2 "error creating upload"
local response
if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then
log 2 "error creating upload: $response"
return 1
fi
upload_id="$response"
if ! list_and_check_upload "$1" "$2" "$upload_id"; then
log 2 "error listing multipart uploads after upload creation"
return 1
tests/util/util_multipart_before_completion.sh
+25 -8
View File
@@ -351,10 +351,11 @@ multipart_upload_range_too_large() {
}
list_and_check_upload() {
if [ $# -lt 2 ]; then
if ! check_param_count_ge_le "bucket, key, upload ID" 2 3 $#; then
log 2 "'list_and_check_upload' requires bucket, key, upload ID (optional)"
return 1
fi
if ! uploads=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/uploads.txt" ./tests/rest_scripts/list_multipart_uploads.sh); then
log 2 "error listing multipart uploads before upload: $result"
return 1
@@ -440,10 +441,14 @@ upload_part_check_etag_header() {
if ! check_param_count_v2 "bucket, key, upload ID, part number, part" 5 $#; then
return 1
fi
if ! etag=$(upload_part_rest "$1" "$2" "$3" "$4" "$5" 2>&1); then
log 2 "error getting etag: $etag"
local response
if ! response=$(upload_part_rest "$1" "$2" "$3" "$4" "$5" 2>&1); then
log 2 "error getting etag: $response"
return 1
fi
etag="$response"
if ! [[ "$etag" =~ ^\"[0-9a-f]+\" ]]; then
log 2 "etag pattern mismatch, etag ($etag) should be hex string surrounded by quotes"
return 1
@@ -456,19 +461,31 @@ upload_part_copy_check_etag_header() {
log 2 "'upload_part_copy_check_etag_header' requires bucket, destination file, part location"
return 1
fi
if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then
log 2 "error creating upload and getting ID: $upload_id"
local response
if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then
log 2 "error creating upload and getting ID: $response"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="1" UPLOAD_ID="$upload_id" PART_LOCATION="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part_copy.sh); then
upload_id="$response"
if ! response=$(get_file_name 2>&1); then
log 2 "error getting file name: $response"
return 1
fi
file_name="$response"
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="1" UPLOAD_ID="$upload_id" PART_LOCATION="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/upload_part_copy.sh); then
# shellcheck disable=SC2154
log 2 "error uploading part: $result"
return 1
fi
if ! etag=$(get_element_text "$TEST_FILE_FOLDER/response.txt" "CopyPartResult" "ETag"); then
log 2 "error getting etag"
if ! response=$(get_element_text "$TEST_FILE_FOLDER/$file_name" "CopyPartResult" "ETag" 2>&1); then
log 2 "error getting etag: $response"
return 1
fi
etag="$response"
log 5 "etag: $etag"
if ! [[ "$etag" =~ ^\"[0-9a-f]+\" ]]; then
log 2 "etag pattern mismatch, etag ($etag) should be hex string surrounded by quotes"
tests/util/util_versioning.sh
+38 -23
View File
@@ -83,15 +83,22 @@ echo_versions() {
if ! check_param_count_v2 "'Version' or 'DeleteMarker', 'Key' or 'VersionId' or 'IsLatest', file" 3 $#; then
return 1
fi
if ! keys=$(xmllint --xpath "//*[local-name()=\"$1\"]/*[local-name()=\"$2\"]/text()" "$3" | xmlstarlet unesc 2>&1); then
if [[ "$keys" == *"XPath set is empty"* ]]; then
local response
if ! response=$(xmllint --xpath "//*[local-name()=\"$1\"]/*[local-name()=\"$2\"]/text()" "$3" 2>&1); then
if [[ "$response" == *"XPath set is empty"* ]]; then
return 0
fi
log 2 "error getting Version 'Key' values: $keys"
log 2 "error getting Version '$1', '$2' values: $response"
return 1
else
keys="$response"
fi
log 5 "keys to append: ${keys[*]}"
echo "${keys[*]}"
unescaped_keys=$(echo -n "$keys" | xmlstarlet unesc)
log 5 "keys to append: ${unescaped_keys[*]}"
echo "${unescaped_keys[*]}"
return 0
}
get_base64_version_keys_and_ids() {
@@ -131,42 +138,50 @@ parse_versions_rest() {
version_keys=()
version_ids=()
version_islatests=()
if ! keys=$(echo_versions "Version" "Key" "$1"); then
log 2 "error getting Version Key values: $keys"
local response
if ! response=$(echo_versions "Version" "Key" "$1" 2>&1); then
log 2 "error getting Version Key values: $response"
return 1
fi
# shellcheck disable=SC2206
version_keys+=($keys)
if ! ids=$(echo_versions "Version" "VersionId" "$1"); then
log 2 "error getting Version VersionId values: $ids"
version_keys+=($response)
if ! response=$(echo_versions "Version" "VersionId" "$1" 2>&1); then
log 2 "error getting Version VersionId values: $response"
return 1
fi
# shellcheck disable=SC2206
version_ids+=($ids)
if ! is_latest=$(echo_versions "Version" "IsLatest" "$1"); then
log 2 "error getting Version IsLatest values: $is_latest"
version_ids+=($response)
if ! response=$(echo_versions "Version" "IsLatest" "$1" 2>&1); then
log 2 "error getting Version IsLatest values: $response"
return 1
fi
# shellcheck disable=SC2206
version_islatests+=($is_latest)
if ! keys=$(echo_versions "DeleteMarker" "Key" "$1"); then
log 2 "error getting DeleteMarker Key values: $keys"
version_islatests+=($response)
if ! response=$(echo_versions "DeleteMarker" "Key" "$1" 2>&1); then
log 2 "error getting DeleteMarker Key values: $response"
return 1
fi
# shellcheck disable=SC2206
version_keys+=($keys)
if ! ids=$(echo_versions "DeleteMarker" "VersionId" "$1"); then
log 2 "error getting DeleteMarker VersionId values: $ids"
version_keys+=($response)
if ! response=$(echo_versions "DeleteMarker" "VersionId" "$1" 2>&1); then
log 2 "error getting DeleteMarker VersionId values: $response"
return 1
fi
# shellcheck disable=SC2206
version_ids+=($ids)
if ! is_latest=$(echo_versions "DeleteMarker" "IsLatest" "$1"); then
log 2 "error getting DeleteMarker IsLatest values: $is_latest"
version_ids+=($response)
if ! response=$(echo_versions "DeleteMarker" "IsLatest" "$1" 2>&1); then
log 2 "error getting DeleteMarker IsLatest values: $response"
return 1
fi
# shellcheck disable=SC2206
version_islatests+=($is_latest)
version_islatests+=($response)
log 5 "version keys: ${version_keys[*]}"
log 5 "version IDs: ${version_ids[*]}"
log 5 "base64 pairs: ${base64_pairs[*]}"