From 32b50dc0ac0d7a711aafc623936e85274ff93c84 Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Tue, 6 Oct 2026 12:55:44 -0300 Subject: [PATCH] test: more multipart tests (ordering, part num, abort, etc. coverage) --- tests/commands/list_parts.sh | 16 ++- .../complete_multipart_upload_rest.sh | 106 +++++++++++++++--- tests/drivers/file.sh | 41 ++++--- tests/drivers/head_object/head_object_rest.sh | 23 ++-- .../list_multipart_uploads_rest.sh | 32 ++++-- tests/drivers/list_parts/list_parts_rest.sh | 53 ++++++++- tests/drivers/rest.sh | 27 ++++- tests/test_rest_head_object.sh | 6 +- tests/test_rest_list_parts.sh | 47 ++++++++ tests/test_rest_multipart.sh | 104 +++++++++++++++-- 10 files changed, 392 insertions(+), 63 deletions(-) diff --git a/tests/commands/list_parts.sh b/tests/commands/list_parts.sh index 93b4a720..6f63fb8d 100644 --- a/tests/commands/list_parts.sh +++ b/tests/commands/list_parts.sh @@ -38,4 +38,18 @@ list_parts_with_user() { listed_parts="$(echo -n "$listed_parts" | grep -v "InsecureRequestWarning")" log 5 "listed parts: $listed_parts" return 0 -} \ No newline at end of file +} + +list_parts_rest() { + if ! check_param_count_gt "bucket name, key, upload ID, callback, other params" 3 $#; then + return 1 + fi + local bucket_name="$1" key="$2" upload_id="$3" callback="$4" params=("${@:5}") + local response + + if ! response=$(send_rest_go_command_callback "200" "$callback" "-bucketName" "$bucket_name" "-objectKey" "$key" "-query" "uploadId=$upload_id" "${params[@]}" 2>&1); then + log 2 "error sending REST ListParts command or checking response: $response" + return 1 + fi + return 0 +} diff --git a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh index 985371b8..dfdf7b6c 100644 --- a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh +++ b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh @@ -149,7 +149,7 @@ test_complete_multipart_upload_unneeded_algorithm_parameter() { fi local checksum_type="$1" algorithm="$2" local file_and_bucket bucket_name mp_file_name - lcoal -a response_lines + local -a response_lines if ! file_and_bucket=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then log 2 "error setting up file and bucket" @@ -394,18 +394,54 @@ create_upload_part_copy_rest() { return 0 } +check_wrong_etag() { + if ! check_param_count_v2 "data file, part number, upload ID, etag" 4 $#; then + return 1 + fi + local data_file="$1" part_number="$2" upload_id="$3" etag="$4" + local response xml_file + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting XML filename: $response" + return 1 + fi + xml_file="$TEST_FILE_FOLDER/$response" + + if ! get_xml_data "$data_file" "$xml_file"; then + log 2 "error getting XML data" + return 1 + fi + if ! check_xml_element "$xml_file" "InvalidPart" "Code"; then + log 2 "code mismatch" + return 1 + fi + if ! check_xml_element "$xml_file" "$upload_id" "UploadId"; then + log 2 "upload ID mismatch" + return 1 + fi + if ! check_xml_element "$xml_file" "$part_number" "PartNumber"; then + log 2 "part number mismatch" + return 1 + fi + if ! check_xml_element "$xml_file" "$etag" "ETag"; then + log 2 "etag mismatch" + return 1 + fi + return 0 +} + create_upload_finish_wrong_etag() { if ! check_param_count "create_upload_finish_wrong_etag" "bucket, key" 2 $#; then return 1 fi local bucket="$1" key="$2" - local etag part_number response parts_payload result result_file error_file upload_id + local etag part_number response parts_payload result result_file upload_id - if ! response=$(get_file_names 2 2>&1); then + if ! response=$(get_file_name 2>&1); then log 2 "error getting file names: $response" return 1 fi - read -r result_file error_file <<< "$response" + result_file="$response" etag="gibberish" part_number=1 @@ -416,7 +452,7 @@ create_upload_finish_wrong_etag() { upload_id="$response" parts_payload="$etag$part_number" - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/$result_file" ./tests/rest_scripts/complete_multipart_upload.sh); then + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/$result_file" ./tests/rest_scripts/complete_multipart_upload.sh 2>&1); then log 2 "error completing multipart upload: $result" return 1 fi @@ -424,24 +460,64 @@ create_upload_finish_wrong_etag() { log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/$result_file")" return 1 fi - if ! get_xml_data "$TEST_FILE_FOLDER/$result_file" "$TEST_FILE_FOLDER/$error_file"; then - log 2 "error getting XML data" + if ! check_wrong_etag "$TEST_FILE_FOLDER/$result_file" "$part_number" "$upload_id" "$etag"; then + log 2 "error checking etag in error message" return 1 fi - if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "InvalidPart" "Code"; then - log 2 "code mismatch" + return 0 +} + +complete_multipart_upload_overwrite_part() { + if ! check_param_count_v2 "bucket, key, file" 3 $#; then return 1 fi - if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "$upload_id" "UploadId"; then - log 2 "upload ID mismatch" + local bucket="$1" key="$2" file="$3" + local response upload_id part_idx etag parts_payload="" + local -a parts + + if ! response=$(split_file "$file" 4 2>&1); then + log 2 "error splitting file: $response" return 1 fi - if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "$part_number" "PartNumber"; then - log 2 "part number mismatch" + read -r -a parts <<< "$response" + if ! response=$(create_multipart_upload_rest "$bucket" "$key" "" "parse_upload_id" 2>&1); then + log 2 "error creating multipart upload: $response" return 1 fi - if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "$etag" "ETag"; then - log 2 "etag mismatch" + upload_id="$response" + + if ! response=$(upload_part_rest "$bucket" "$key" "$upload_id" 1 "${parts[1]}" 2>&1); then + log 2 "error uploading part 2 as part 1: $response" + return 1 + fi + for ((part_idx=1; part_idx<=${#parts[@]}; part_idx++)); do + if ! response=$(upload_part_rest "$bucket" "$key" "$upload_id" "$part_idx" "${parts[$((part_idx-1))]}" 2>&1); then + log 2 "error uploading part '$part_idx': $response" + return 1 + fi + etag="$response" + parts_payload+="$etag$part_idx" + done + if ! response=$(complete_multipart_upload_rest "$bucket" "$key" "$upload_id" "$parts_payload" 2>&1); then + log 2 "error completing multipart upload: $response" + return 1 + fi + if ! download_and_compare_file "$file" "$bucket" "$key" "${file}-copy"; then + log 2 "error downloading and comparing file" + return 1 + fi + return 0 +} + +complete_multipart_upload_expect_invalid_part_error_callback() { + if ! check_param_count_v2 "bucket name, key, upload ID, parts payload, expected part number, expected etag" 6 $#; then + return 1 + fi + local bucket_name="$1" key="$2" upload_id="$3" parts_payload="$4" expected_part_number="$5" expected_etag="$6" + + if ! send_rest_command_expect_error_callback "BUCKET_NAME=$bucket_name OBJECT_KEY=$key UPLOAD_ID=$upload_id PARTS=$parts_payload" \ + "./tests/rest_scripts/complete_multipart_upload.sh" "400" "InvalidPart" "One or more" "check_wrong_etag" "$expected_part_number" "$upload_id" "$expected_etag"; then + log 2 "error checking complete multipart upload command" return 1 fi return 0 diff --git a/tests/drivers/file.sh b/tests/drivers/file.sh index f35529fd..03e70c1b 100644 --- a/tests/drivers/file.sh +++ b/tests/drivers/file.sh @@ -162,17 +162,9 @@ setup_bucket_and_large_file_v2() { return 0 } -setup_bucket_and_large_file_v3() { - if ! check_param_count_ge_le "bucket env var, file size in MBs (optional)" 1 2 $#; then - return 1 - fi - - local response bucket_name file_name error - if ! response=$(setup_bucket_v3 "$1" 2>&1); then - log 2 "error setting up bucket: $response" - return 1 - fi - bucket_name="$response" +setup_large_file_v3() { + local file_size="$1" + local response file_name if ! response=$(get_file_name 2>&1); then log 2 "error getting file name: $response" @@ -180,10 +172,33 @@ setup_bucket_and_large_file_v3() { fi file_name="$response" - if ! error=$(create_large_file "$file_name" "$2" 2>&1); then - log 2 "error creating large file: $error" + if ! response=$(create_large_file "$file_name" "$file_size" 2>&1); then + log 2 "error creating large file: $response" return 1 fi + echo "$file_name" + return 0 +} + +setup_bucket_and_large_file_v3() { + if ! check_param_count_ge_le "bucket env var, file size in MBs (optional)" 1 2 $#; then + return 1 + fi + local bucket_env_var="$1" file_size="$2" + local response bucket_name file_name error + + if ! response=$(setup_bucket_v3 "$bucket_env_var" 2>&1); then + log 2 "error setting up bucket: $response" + return 1 + fi + bucket_name="$response" + + if ! response=$(setup_large_file_v3 "$file_size" 2>&1); then + log 2 "error setting up large file: $response" + return 1 + fi + file_name="$response" + echo "$bucket_name $file_name" return 0 } diff --git a/tests/drivers/head_object/head_object_rest.sh b/tests/drivers/head_object/head_object_rest.sh index 69391bbf..cbaf6ca3 100644 --- a/tests/drivers/head_object/head_object_rest.sh +++ b/tests/drivers/head_object/head_object_rest.sh @@ -261,17 +261,24 @@ head_object_check_header_key_and_value() { } check_header_partial_content_response() { - if ! check_param_count_v2 "header data, part number, full object size, part size" 4 $#; then + if ! check_param_count_v2 "header data, part number, full object size, part one size, part size" 5 $#; then return 1 fi - log 5 "header: $(cat "$1")" - starting_byte=$((($2-1)*$4)) - ending_byte=$(($2*$4-1)) - if [ "$3" -lt "$ending_byte" ]; then - ending_byte="$(($3-1))" + local header_data="$1" part_number="$2" full_object_size="$3" part_one_size="$4" part_size="$5" + local starting_byte ending_byte content_range_string result + + log 5 "header: $(cat "$header_data")" + starting_byte=$(((part_number-1)*part_one_size)) + ending_byte=$((part_number*part_one_size-1)) + if [ "$full_object_size" -lt "$ending_byte" ]; then + ending_byte="$((full_object_size-1))" fi - content_range_string="bytes $starting_byte-$ending_byte/$3" - if ! result=$(check_for_header_key_and_value "$1" "Content-Range" "$content_range_string" 2>&1); then + if [ "$part_size" -ne "$((ending_byte-starting_byte+1))" ]; then + log 2 "mismatch between part size '$part_size' and byte range '$starting_byte'-'$ending_byte'" + return 1 + fi + content_range_string="bytes ${starting_byte}-${ending_byte}/${full_object_size}" + if ! result=$(check_for_header_key_and_value "$header_data" "Content-Range" "$content_range_string" 2>&1); then log 2 "error checking for header key and value: $result" return 1 fi diff --git a/tests/drivers/list_multipart_uploads/list_multipart_uploads_rest.sh b/tests/drivers/list_multipart_uploads/list_multipart_uploads_rest.sh index 7cdfdf0e..0d9d593a 100644 --- a/tests/drivers/list_multipart_uploads/list_multipart_uploads_rest.sh +++ b/tests/drivers/list_multipart_uploads/list_multipart_uploads_rest.sh @@ -86,15 +86,16 @@ get_next_values() { if ! check_param_count_v2 "data file" 1 $#; then return 1 fi + local data_file="$1" local response next_key_marker next_upload_id_marker - if ! response=$(get_element_text "$1" "NextKeyMarker" 2>&1); then + if ! response=$(get_element_text "$data_file" "NextKeyMarker" 2>&1); then log 2 "error checking NextKeyMarker: $response" return 1 fi next_key_marker="$response" - if ! response=$(get_element_text "$1" "NextUploadIdMarker" 2>&1); then + if ! response=$(get_element_text "$data_file" "NextUploadIdMarker" 2>&1); then log 2 "error checking NextUploadIdMarker: $response" return 1 fi @@ -109,17 +110,32 @@ list_multipart_uploads_get_check_next_values() { if ! check_param_count_ge_le "bucket, expected key, expected upload ID, next key (optional), next upload ID (optional)" 3 5 $#; then return 1 fi - local query_val="max-uploads=1&uploads" response - if [ "$4" != "" ]; then - query_val+="&key-marker=$4" + local bucket="$1" expected_key="$2" expected_upload_id="$3" next_key="$4" next_upload_id="$5" + local query_val="max-uploads=1&uploads" response key_marker upload_id_marker + local -a response_lines + + if [ -n "$next_key" ]; then + query_val+="&key-marker=${next_key}" fi - if [ "$5" != "" ]; then - query_val+="&upload-id-marker=$5" + if [ -n "$next_upload_id" ]; then + query_val+="&upload-id-marker=${next_upload_id}" fi - if ! response=$(send_rest_go_command_callback "200" "get_next_values" "-bucketName" "$1" "-query" "$query_val" 2>&1); then + if ! response=$(send_rest_go_command_callback "200" "get_next_values" "-bucketName" "$bucket" "-query" "$query_val" 2>&1); then log 2 "error sending multipart upload and checking key and upload ID vals" return 1 fi + mapfile -t response_lines <<< "$response" + key_marker="${response_lines[0]}" + upload_id_marker="${response_lines[1]}" + + if [ "$expected_key" != "$key_marker" ]; then + log 2 "expected key marker '$expected_key', was '$key_marker'" + return 1 + fi + if [ "$expected_upload_id" != "$upload_id_marker" ]; then + log 2 "expected upload ID marker '$expected_upload_id', was '$upload_id_marker'" + return 1 + fi return 0 } diff --git a/tests/drivers/list_parts/list_parts_rest.sh b/tests/drivers/list_parts/list_parts_rest.sh index 95c07412..01cc0a03 100644 --- a/tests/drivers/list_parts/list_parts_rest.sh +++ b/tests/drivers/list_parts/list_parts_rest.sh @@ -50,7 +50,7 @@ check_part_list_rest() { log 2 "error getting file name: $file_name" return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_parts.sh); then + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_parts.sh 2>&1); then log 2 "error listing multipart upload parts: $result" return 1 fi @@ -241,3 +241,54 @@ list_parts_check_with_marker_and_max_parts() { fi return 0 } + +list_parts_check_part_ordering() { + if ! check_param_count_gt "data file, bucket, key, upload ID, pairs of part numbers/etags" 3 $#; then + return 1 + fi + if [ "$(($# % 2))" -ne 0 ]; then + log 2 "must be an even param count" + return 1 + fi + local data_file="$1" bucket="$2" key="$3" upload_id="$4" part_etag_pairs=("${@:5}") + local response + local -a parts + + log 5 "data: $(cat "$data_file")" + + if ! check_xml_element "$data_file" "$bucket" "ListPartsResult" "Bucket"; then + log 2 "error checking 'Bucket' element" + return 1 + fi + if ! check_xml_element "$data_file" "$key" "ListPartsResult" "Key"; then + log 2 "error checking 'Key' element" + return 1 + fi + if ! check_xml_element "$data_file" "$upload_id" "ListPartsResult" "UploadId"; then + log 2 "error checking 'UploadId' element" + return 1 + fi + if ! response=$(get_element "$data_file" "ListPartsResult" "Part" 2>&1); then + log 2 "error getting parts: $response" + return 1 + fi + mapfile -t parts <<< "$response" + if [ "${#parts[@]}" -ne "$((${#part_etag_pairs[@]}/2))" ]; then + log 2 "expected '$((${#part_etag_pairs[@]}/2))' parts, was '${#parts[@]}'" + return 1 + fi + pair_idx=0 + for part in "${parts[@]}"; do + if ! check_xml_element_inside_string "$part" "${part_etag_pairs[$pair_idx]}" "Part" "PartNumber"; then + log 2 "error checking 'PartNumber' element" + return 1 + fi + ((pair_idx++)) + if ! check_xml_element_inside_string "$part" "${part_etag_pairs[$pair_idx]}" "Part" "ETag"; then + log 2 "error checking 'PartNumber' element" + return 1 + fi + ((pair_idx++)) + done + return 0 +} diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index e36541d2..d3edd604 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -92,6 +92,7 @@ send_rest_command() { return 1 fi local response output_file response_code + local -a env_var_array if [[ "$1" == *"OUTPUT_FILE"* ]]; then if ! response=$(echo -n "$1" | sed -n 's/^.*OUTPUT_FILE=\([^ ]*\).*$/\1/p' 2>&1); then @@ -108,8 +109,8 @@ send_rest_command() { fi local env_array=("env" "COMMAND_LOG=$COMMAND_LOG" "OUTPUT_FILE=$output_file") if [ "$1" != "" ]; then - IFS=' ' read -r -a env_vars <<< "$1" - env_array+=("${env_vars[@]}") + IFS=' ' read -r -a env_var_array <<< "$1" + env_array+=("${env_var_array[@]}") fi # shellcheck disable=SC2068 if ! response=$(${env_array[@]} "$2" 2>&1); then @@ -130,9 +131,23 @@ send_rest_command_expect_error() { if ! check_param_count_v2 "env vars, script, response code, error, message" 5 $#; then return 1 fi + local env_vars="$1" script="$2" response_code="$3" error="$4" message="$5" + + if ! send_rest_command_expect_error_callback "$env_vars" "$script" "$response_code" "$error" "$message"; then + log 2 "error checking REST error with empty callback" + return 1 + fi + return 0 +} + +send_rest_command_expect_error_callback() { + if ! check_param_count_gt "env vars, script, expected HTTP code, expected error, expected message, callback, params" 5 $#; then + return 1 + fi + local env_vars="$1" script="$2" expected_http_code="$3" expected_error="$4" expected_message="$5" callback="$6" params=("${@:7}") local response response_lines response_code output_file - if ! response=$(send_rest_command "$1" "$2" 2>&1); then + if ! response=$(send_rest_command "$env_vars" "$script" 2>&1); then log 2 "error sending REST command: $response" return 1 fi @@ -140,10 +155,14 @@ send_rest_command_expect_error() { response_code="${response_lines[0]}" output_file="${response_lines[1]}" - if ! check_rest_expected_error "$response_code" "$output_file" "$3" "$4" "$5"; then + if ! check_rest_expected_error "$response_code" "$output_file" "$expected_http_code" "$expected_error" "$expected_message"; then log 2 "error checking REST error" return 1 fi + if [ -n "$callback" ] && ! "$callback" "$output_file" "${params[@]}"; then + log 2 "callback error" + return 1 + fi return 0 } diff --git a/tests/test_rest_head_object.sh b/tests/test_rest_head_object.sh index 2758b9db..41af9758 100755 --- a/tests/test_rest_head_object.sh +++ b/tests/test_rest_head_object.sh @@ -88,16 +88,16 @@ source ./tests/drivers/string.sh assert_success part_size_one="$output" - run get_file_size "$part_one" + run get_file_size "$part_two" assert_success part_size_two="$output" run send_rest_go_command_callback "206" "check_header_partial_content_response" "-bucketName" "$bucket_name" "-objectKey" "$test_file" \ - "-method" "HEAD" "-query" "partNumber=1" "--" "1" "$file_bytes" "$part_size_one" + "-method" "HEAD" "-query" "partNumber=1" "--" "1" "$file_bytes" "$part_size_one" "$part_size_one" assert_success run send_rest_go_command_callback "206" "check_header_partial_content_response" "-bucketName" "$bucket_name" "-objectKey" "$test_file" \ - "-method" "HEAD" "-query" "partNumber=2" "--" "2" "$file_bytes" "$part_size_two" + "-method" "HEAD" "-query" "partNumber=2" "--" "2" "$file_bytes" "$part_size_one" "$part_size_two" assert_success } diff --git a/tests/test_rest_list_parts.sh b/tests/test_rest_list_parts.sh index 46fad97b..9cd4b701 100755 --- a/tests/test_rest_list_parts.sh +++ b/tests/test_rest_list_parts.sh @@ -84,3 +84,50 @@ source ./tests/setup.sh run list_parts_check_with_marker_and_max_parts "$bucket_name" "$file_name" "$upload_id" 1 4 0 assert_success } + +@test "REST - ListParts - out of order" { + local etag_one etag_two etag_three + + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run create_multipart_upload_rest "$bucket_name" "$file_name" "" "parse_upload_id" + assert_success + upload_id=$output + + run split_file "$TEST_FILE_FOLDER/$file_name" 3 + assert_success + read -r -a parts <<< "$output" + + run upload_part_rest "$bucket_name" "$file_name" "$upload_id" "7" "${parts[0]}" + assert_success + etag_one="$output" + log 5 "sha256sum: $(sha256sum "${parts[0]}")" + + run upload_part_rest "$bucket_name" "$file_name" "$upload_id" "2" "${parts[0]}" + assert_success + etag_two="$output" + + run upload_part_rest "$bucket_name" "$file_name" "$upload_id" "10000" "${parts[0]}" + assert_success + etag_three="$output" + + run list_parts_rest "$bucket_name" "$file_name" "$upload_id" "list_parts_check_part_ordering" "--" "$bucket_name" "$file_name" "$upload_id" \ + "2" "$etag_two" "7" "$etag_one" "10000" "$etag_three" + assert_success + + offset=100 + old=$(od -An -tu1 -N1 -j "$offset" "${parts[0]}") + new=$((old ^ 255)) + printf "\\$(printf '%03o' "$new")" | dd of="${parts[0]}" bs=1 seek="$offset" conv=notrunc status=none + log 5 "sha256sum: $(sha256sum "${parts[0]}")" + + run upload_part_rest "$bucket_name" "$file_name" "$upload_id" "7" "${parts[0]}" + assert_success + etag_one="$output" + + run list_parts_rest "$bucket_name" "$file_name" "$upload_id" "list_parts_check_part_ordering" "--" "$bucket_name" "$file_name" "$upload_id" \ + "2" "$etag_two" "7" "$etag_one" "10000" "$etag_three" + assert_success +} \ No newline at end of file diff --git a/tests/test_rest_multipart.sh b/tests/test_rest_multipart.sh index bcbfccc5..65c3ef71 100755 --- a/tests/test_rest_multipart.sh +++ b/tests/test_rest_multipart.sh @@ -64,17 +64,12 @@ source ./tests/drivers/upload_part/upload_part_rest.sh } # tags: curl,multipart,CompleteMultipartUpload,invalid-header,ETag -@test "REST - complete upload - invalid part" { - run get_bucket_name "$BUCKET_ONE_NAME" - assert_success - bucket_name="$output" +@test "REST - complete upload - finish with non-existent part" { + local bucket_name large_test_file - run get_file_name - assert_success - large_test_file="$output" - - run setup_bucket_and_large_file_v2 "$bucket_name" "$large_test_file" + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" assert_success + read -r bucket_name large_test_file <<< "$output" run create_upload_finish_wrong_etag "$bucket_name" "$large_test_file" assert_success @@ -377,7 +372,7 @@ source ./tests/drivers/upload_part/upload_part_rest.sh # tags: curl,multipart,checksum,CompleteMultipartUpload,invalid-header,x-amz-checksum-type,x-amz-checksum-algorithm @test "REST - multipart - composite - incorrect crc32" { - run test_complete_multipart_upload_incorrect_checksum "COMPOSITE" "CRC32C" + run test_complete_multipart_upload_incorrect_checksum "COMPOSITE" "CRC32" assert_success } @@ -475,3 +470,92 @@ source ./tests/drivers/upload_part/upload_part_rest.sh run complete_multipart_upload_empty_upload_id "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/$file_name" assert_success } + +@test "REST - multipart overwrite" { + local bucket_name file_name + + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run complete_multipart_upload_overwrite_part "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/$file_name" + assert_success +} + +@test "REST - multipart - InvalidPart error returns correct etag" { + if [ "$DIRECT" == "true" ]; then + skip "https://github.com/versity/versitygw/issues/2525" + fi + local bucket_name file_name upload_id parts_payload bad_etag first_etag bad_parts_payload + + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run multipart_upload_rest_before_completion "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/$file_name" 2 + assert_success + read -r upload_id parts_payload <<< "$output" + + bad_etag="00000000000000000000000000000000" + first_etag=${parts_payload#*} + first_etag=${first_etag%%*} + bad_parts_payload=${parts_payload/"$first_etag"/"$bad_etag"} + + run complete_multipart_upload_expect_invalid_part_error_callback "$bucket_name" "$file_name" "$upload_id" "$bad_parts_payload" "1" "$bad_etag" + assert_success +} + +@test "REST - multipart failure, then success" { + local bucket_name file_name upload_id parts_payload first_etag bad_parts_payload + + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_name <<< "$output" + + run multipart_upload_rest_before_completion "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/$file_name" 2 + assert_success + read -r upload_id parts_payload <<< "$output" + + first_etag=${parts_payload#*} + first_etag=${first_etag%%*} + bad_parts_payload=${parts_payload/"$first_etag"/"00000000000000000000000000000000"} + + run complete_multipart_upload_rest "$bucket_name" "$file_name" "$upload_id" "$bad_parts_payload" + assert_failure + + run complete_multipart_upload_rest "$bucket_name" "$file_name" "$upload_id" "$parts_payload" + assert_success +} + +@test "REST - multipart abort isolation" { + local bucket_name file_one_name file_two_name key_name upload_id_one upload_id_two parts_payload_two + + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" + assert_success + read -r bucket_name file_one_name <<< "$output" + + run setup_large_file_v3 + assert_success + file_two_name="$output" + + run get_file_name + assert_success + key_name="$output" + + run multipart_upload_rest_before_completion "$bucket_name" "$key_name" "$TEST_FILE_FOLDER/$file_one_name" 2 + assert_success + read -r upload_id_one _ <<< "$output" + + run multipart_upload_rest_before_completion "$bucket_name" "$key_name" "$TEST_FILE_FOLDER/$file_two_name" 2 + assert_success + read -r upload_id_two parts_payload_two <<< "$output" + + run abort_multipart_upload_rest "$bucket_name" "$key_name" "$upload_id_one" + assert_success + + run complete_multipart_upload_rest "$bucket_name" "$key_name" "$upload_id_two" "$parts_payload_two" + assert_success + + run download_and_compare_file "$TEST_FILE_FOLDER/$file_two_name" "$bucket_name" "$key_name" "$TEST_FILE_FOLDER/${file_two_name}-copy" + assert_success +}