mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-20 15:04:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: chunked upload - invalid checksums, retention, legal hold tests

Browse Source
This commit is contained in:
Luke McCrone
2025-04-06 22:24:26 -03:00
parent 7a3de637e4
commit 8431ba2a21
25 changed files with 812 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/util/util_attributes.sh
+1 -1
View File
@@ -87,7 +87,7 @@ add_and_check_checksum() {
log 2 "'add_and_check_checksum' requires data file, key"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$BUCKET_ONE_NAME" OBJECT_KEY="$2" CHECKSUM="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then
if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$BUCKET_ONE_NAME" OBJECT_KEY="$2" CHECKSUM_TYPE="sha256" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then
log 2 "error sending object file: $result"
return 1
fi
tests/util/util_bucket.sh
+1 -1
View File
@@ -278,7 +278,7 @@ setup_bucket() {
# bucket creation and resets take longer to propagate in direct mode
if [ "$DIRECT" == "true" ]; then
sleep 10
sleep 15
fi
if [[ $1 == "s3cmd" ]]; then
tests/util/util_chunked_upload.sh
+152 -54
View File
@@ -5,12 +5,18 @@ attempt_seed_signature_without_content_length() {
log 2 "'attempt_seed_signature_without_content_length' requires bucket name, key, data file"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" CONTENT_ENCODING="aws-chunked" BUCKET_NAME="$1" OBJECT_KEY="$2" DATA_FILE="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then
log 2 "error putting object: $result"
if ! result=$(COMMAND_LOG="$COMMAND_LOG" \
AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \
DATA_FILE="$1" \
BUCKET_NAME="$2" \
OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" NO_CONTENT_LENGTH="true" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then
log 2 "error creating command: $result"
return 1
fi
if [ "$result" != 411 ]; then
log 2 "expected '411', actual '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 411; then
log 2 "error in sending or checking response code"
return 1
fi
return 0
@@ -32,12 +38,8 @@ attempt_chunked_upload_with_bad_first_signature() {
return 1
fi
host="${AWS_ENDPOINT_URL#http*://}"
if [ "$host" == "s3.amazonaws.com" ]; then
host+=":443"
fi
if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then
log 2 "error sending openssl command: $result"
if ! result=$(send_via_openssl "$TEST_FILE_FOLDER/command.txt"); then
log 2 "error sending command via openssl"
return 1
fi
response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
@@ -46,7 +48,9 @@ attempt_chunked_upload_with_bad_first_signature() {
log 2 "expected code '403', was '$response_code'"
return 1
fi
response_data="$(echo "$result" | grep "<")"
log 5 "result: $result"
response_data="$(echo "$result" | grep "<Error>")"
response_data="${response_data/---/}"
log 5 "response data: $response_data"
log 5 "END"
if ! check_xml_element <(echo "$response_data") "SignatureDoesNotMatch" "Error" "Code"; then
@@ -72,17 +76,8 @@ chunked_upload_success() {
return 1
fi
host="${AWS_ENDPOINT_URL#http*://}"
if [ "$host" == "s3.amazonaws.com" ]; then
host+=":443"
fi
if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then
log 2 "error sending openssl command: $result"
return 1
fi
response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
if [ "$response_code" != "200" ]; then
log 2 "expected response '200', was '$response_code'"
if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then
log 2 "error sending command via openssl or checking response code"
return 1
fi
return 0
@@ -107,12 +102,8 @@ attempt_chunked_upload_with_bad_final_signature() {
log 2 "error creating command: $result"
return 1
fi
host="${AWS_ENDPOINT_URL#http*://}"
if [ "$host" == "s3.amazonaws.com" ]; then
host+=":443"
fi
if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then
log 2 "error sending openssl command: $result"
if ! result=$(send_via_openssl "$TEST_FILE_FOLDER/command.txt"); then
log 2 "error sending command via openssl"
return 1
fi
response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
@@ -144,25 +135,15 @@ put_object_chunked_trailer_success() {
DATA_FILE="$1" \
BUCKET_NAME="$2" \
OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false TRAILER="x-amz-checksum-$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then
log 2 "error creating command: $result"
return 1
fi
log 2 "error creating command: $result"
return 1
fi
host="${AWS_ENDPOINT_URL#http*://}"
if [ "$host" == "s3.amazonaws.com" ]; then
host+=":443"
fi
if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then
log 2 "error sending openssl command: $result"
return 1
fi
log 5 "result: $result"
response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
if [ "$response_code" != "200" ]; then
log 2 "expected response '200', was '$response_code'"
return 1
fi
return 0
if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then
log 2 "error sending command via openssl or checking response code"
return 1
fi
return 0
}
put_chunked_upload_trailer_invalid() {
@@ -182,12 +163,8 @@ put_chunked_upload_trailer_invalid() {
return 1
fi
host="${AWS_ENDPOINT_URL#http*://}"
if [ "$host" == "s3.amazonaws.com" ]; then
host+=":443"
fi
if ! result=$(openssl s_client -connect "$host" -ign_eof < "$TEST_FILE_FOLDER/command.txt" 2>&1); then
log 2 "error sending openssl command: $result"
if ! result=$(send_via_openssl "$TEST_FILE_FOLDER/command.txt"); then
log 2 "error sending command via openssl"
return 1
fi
response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
@@ -196,8 +173,9 @@ put_chunked_upload_trailer_invalid() {
return 1
fi
error_data="$(echo "$result" | grep "<Error>")"
if ! check_xml_element <(echo -n "$error_data") "InvalidRequest" "Error" "Code"; then
log 2 "error checking xml element"
echo -n "$error_data" > "$TEST_FILE_FOLDER/error-data.txt"
if ! check_xml_error_contains "$TEST_FILE_FOLDER/error-data.txt" "InvalidRequest" "The value specified in the x-amz-trailer header is not supported"; then
log 2 "error checking xml error, message"
return 1
fi
return 0
@@ -227,3 +205,123 @@ chunked_upload_trailer_success() {
fi
return 0
}
chunked_upload_trailer_invalid_checksum() {
if [ "$#" -ne 1 ]; then
log 2 "'chunked_upload_trailer_invalid_checksum' requires checksum"
return 1
fi
if ! setup_bucket "s3api" "$BUCKET_ONE_NAME"; then
log 2 "error setting up bucket"
return 1
fi
test_file="test-file"
if ! create_test_file "$test_file" 10000; then
log 2 "error creating test file"
return 1
fi
# shellcheck disable=SC2097,SC2098
if ! result=$(COMMAND_LOG="$COMMAND_LOG" \
AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \
DATA_FILE="$TEST_FILE_FOLDER/$test_file" \
BUCKET_NAME="$BUCKET_ONE_NAME" \
OBJECT_KEY="$test_file" CHUNK_SIZE=8192 TEST_MODE=false TRAILER="x-amz-checksum-$1" CHECKSUM="a" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then
log 2 "error creating command: $result"
return 1
fi
if ! send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/command.txt" "400" "InvalidRequest" "Value for x-amz-checksum-$1 trailing header is invalid."; then
log 2 "error sending openssl and checking response"
return 1
fi
return 0
}
chunked_upload_trailer_incorrect_checksum() {
if [ "$#" -ne 1 ]; then
log 2 "'chunked_upload_trailer_invalid_checksum' requires checksum"
return 1
fi
if ! setup_bucket "s3api" "$BUCKET_ONE_NAME"; then
log 2 "error setting up bucket"
return 1
fi
test_file="test-file"
if ! create_test_file "$test_file" 10000; then
log 2 "error creating test file"
return 1
fi
if ! checksum=$(calculate_incorrect_checksum "$1" "$test_file"); then
log 2 "error calculating incorrect checksum"
return 1
fi
# shellcheck disable=SC2097,SC2098
if ! result=$(COMMAND_LOG="$COMMAND_LOG" \
AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \
DATA_FILE="$TEST_FILE_FOLDER/$test_file" \
BUCKET_NAME="$BUCKET_ONE_NAME" \
OBJECT_KEY="$test_file" CHUNK_SIZE=8192 TEST_MODE=false TRAILER="x-amz-checksum-$1" CHECKSUM="$checksum" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then
log 2 "error creating command: $result"
return 1
fi
uppercase_type="$(echo "$1" | tr '[:lower:]' '[:upper:]')"
if ! send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/command.txt" "400" "BadDigest" "The $uppercase_type you specified did not match the calculated checksum."; then
log 2 "error sending openssl and checking response"
return 1
fi
return 0
}
send_via_openssl() {
if [ "$#" -ne 1 ]; then
log 2 "'send_via_openssl' requires command file"
return 1
fi
host="${AWS_ENDPOINT_URL#http*://}"
if [ "$host" == "s3.amazonaws.com" ]; then
host+=":443"
fi
if ! result=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then
log 2 "error sending openssl command: $result"
return 1
fi
echo "$result"
}
send_via_openssl_and_check_code() {
if [ "$#" -ne 2 ]; then
log 2 "'send_via_openssl_and_check_code' requires command file, expected code"
return 1
fi
if ! result=$(send_via_openssl "$1"); then
log 2 "error sending command via openssl"
return 1
fi
response_code="$(echo "$result" | grep "HTTP/" | awk '{print $2}')"
if [ "$response_code" != "$2" ]; then
log 2 "expected '$2', actual '$response_code' (error response: '$result')"
return 1
fi
echo "$result"
}
send_via_openssl_check_code_error_contains() {
if [ "$#" -ne 4 ]; then
log 2 "'send_via_openssl_check_code_error_contains' requires command file, expected code, error, message"
return 1
fi
if ! result=$(send_via_openssl_and_check_code "$1" "$2"); then
log 2 "error sending and checking code"
return 1
fi
error_data="$(echo "$result" | grep "<Error>" | sed 's/---//')"
echo -n "$error_data" > "$TEST_FILE_FOLDER/error-data.txt"
if ! check_xml_error_contains "$TEST_FILE_FOLDER/error-data.txt" "$3" "$4"; then
log 2 "error checking xml error, message"
return 1
fi
return 0
}
tests/util/util_head_object.sh
+27
View File
@@ -192,3 +192,30 @@ head_object_without_and_with_checksum() {
fi
return 0
}
check_default_checksum() {
if [ $# -ne 3 ]; then
log 2 "'head_object_without_checksum' requires bucket, file, local file"
return 1
fi
if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" ./tests/rest_scripts/head_object.sh); then
log 2 "error getting result: $result"
return 1
fi
if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" CHECKSUM="true" ./tests/rest_scripts/head_object.sh); then
log 2 "error getting result: $result"
return 1
fi
log 5 "result: $(cat "$TEST_FILE_FOLDER/result.txt")"
head_checksum=$(grep -i "x-amz-checksum-crc64nvme" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//')
log 5 "checksum: $head_checksum"
if ! checksum=$(CHECKSUM_TYPE="crc64nvme" DATA_FILE="$3" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh); then
log 2 "error calculating local checksum: $checksum"
return 1
fi
if [ "$head_checksum" != "$checksum" ]; then
log 2 "checksum mismatch (returned: '$head_checksum', local: '$checksum')"
return 1
fi
return 0
}
tests/util/util_legal_hold.sh
+70
View File
@@ -57,3 +57,73 @@ check_legal_hold_without_lock_enabled() {
fi
return 0
}
check_remove_legal_hold_versions() {
if [ $# -ne 3 ]; then
log 2 "'check_remove_legal_hold_versions' requires bucket, key, version ID"
return 1
fi
if ! legal_hold=$(get_object_legal_hold_version_id "$1" "$2" "$3"); then
if [[ "$legal_hold" != *"MethodNotAllowed"* ]]; then
log 2 "error getting object legal hold status with version id"
return 1
fi
return 0
fi
log 5 "legal hold: $legal_hold"
if ! status="$(echo "$legal_hold" | grep -v "InsecureRequestWarning" | jq -r '.LegalHold.Status' 2>&1)"; then
log 2 "error getting legal hold status: $status"
return 1
fi
if [ "$status" == "ON" ]; then
if ! put_object_legal_hold_version_id "$1" "$2" "$3" "OFF"; then
log 2 "error removing legal hold of version ID"
return 1
fi
fi
return 0
}
check_legal_hold_without_payload() {
if [ $# -ne 2 ]; then
log 2 "'check_legal_hold_without_payload' requires bucket name, key"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OMIT_PAYLOAD="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_legal_hold.sh); then
log 2 "error: $result"
return 1
fi
if [ "$result" != "400" ]; then
log 2 "expected '400', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
if ! check_xml_error_contains "$TEST_FILE_FOLDER/result.txt" "MalformedXML" "The XML you provided"; then
log 2 "error checking xml error, message"
return 1
fi
return 0
}
rest_check_legal_hold() {
if [ $# -ne 2 ]; then
log 2 "'rest_check_legal_hold' requires bucket name, key"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" STATUS="ON" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_legal_hold.sh); then
log 2 "error: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/get_object_legal_hold.sh); then
log 2 "error: $result"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/result.txt" "ON" "LegalHold" "Status"; then
log 2 "error checking legal hold status"
return 1
fi
return 0
}
tests/util/util_object.sh
+2 -1
View File
@@ -336,7 +336,7 @@ put_object_rest_checksum() {
check_checksum_rest_invalid() {
if [ $# -ne 1 ]; then
log 2 "'put_object_rest_sha256_invalid' requires checksum type"
log 2 "'check_checksum_rest_invalid' requires checksum type"
return 1
fi
test_file="test_file"
@@ -405,6 +405,7 @@ calculate_incorrect_checksum() {
log 2 "invalid checksum type: $1"
return 1
esac
echo "$incorrect_checksum"
return 0
}
tests/util/util_retention.sh
+21
View File
@@ -92,3 +92,24 @@ log_worm_protection() {
# shellcheck disable=SC2154
log 5 "RETENTION: $retention"
}
retention_rest_without_request_body() {
if [ $# -ne 2 ]; then
log 2 "'retention_rest_without_request_body' requires bucket name, key"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OMIT_PAYLOAD="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object_retention.sh); then
log 2 "error: $result"
return 1
fi
if [ "$result" != "400" ]; then
log 2 "expected '400', was '$result'"
return 1
fi
log 5 "result: $result ($(cat "$TEST_FILE_FOLDER/result.txt"))"
if ! check_xml_error_contains "$TEST_FILE_FOLDER/result.txt" "MalformedXML" "The XML you provided"; then
log 2 "error checking xml reply"
return 1
fi
return 0
}
tests/util/util_setup.sh
+20
View File
@@ -77,3 +77,23 @@ setup_bucket_file_and_user() {
echo "$result"
return 0
}
setup_bucket_object_lock_enabled() {
if [ $# -ne 1 ]; then
log 2 "'setup_bucket_object_lock_enabled' requires bucket name"
return 1
fi
if ! bucket_cleanup_if_bucket_exists "s3api" "$1"; then
log 2 "error cleaning up bucket"
return 1
fi
# in static bucket config, bucket will still exist
if ! bucket_exists "rest" "$1"; then
if ! create_bucket_object_lock_enabled "$1"; then
log 2 "error creating bucket with object lock enabled"
return 1
fi
fi
return 0
}
tests/util/util_versioning.sh
+7 -1
View File
@@ -65,8 +65,14 @@ delete_object_version_with_or_without_retention() {
log 5 "version ID: ${version_ids[$idx]}"
# shellcheck disable=SC2154
if [ "$lock_config_exists" == "true" ]; then
if ! check_remove_legal_hold_versions "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
log 2 "error checking, removing legal hold versions"
fi
if ! put_object_legal_hold_version_id "$1" "${version_keys[$idx]}" "${version_ids[$idx]}" "OFF"; then
log 2 "error turning off object legal hold"
fi
if ! delete_object_version_bypass_retention "$1" "${version_keys[$idx]}" "${version_ids[$idx]}"; then
log 2 "error deleting object version"
log 2 "error deleting object version, bypassing retention"
return 1
fi
else
tests/util/util_xml.sh
+51 -7
View File
@@ -1,17 +1,29 @@
#!/usr/bin/env bash
get_element_text() {
if [ $# -lt 2 ]; then
log 2 "'get_element_text' requires data source, XML tree"
return 1
fi
local xpath='//'
for tree_val in "${@:2}"; do
xpath+='*[local-name()="'$tree_val'"]/'
done
xpath+='text()'
if ! xml_val=$(xmllint --xpath "$xpath" "$1" 2>&1); then
log 2 "error getting XML value matching $xpath: $xml_val (file data: $(cat "$1"))"
return 1
fi
echo "$xml_val"
}
check_xml_element() {
if [ $# -lt 3 ]; then
log 2 "'check_xml_element' requires data source, expected value, XML tree"
return 1
fi
local xpath='//'
for tree_val in "${@:3}"; do
xpath+='*[local-name()="'$tree_val'"]/'
done
xpath+='text()'
if ! xml_val=$(xmllint --xpath "$xpath" "$1" 2>&1); then
log 2 "error getting XML value matching $xpath: $xml_val"
if ! xml_val=$(get_element_text "$1" "${@:3}"); then
log 2 "error getting element text"
return 1
fi
if [ "$2" != "$xml_val" ]; then
@@ -20,3 +32,35 @@ check_xml_element() {
fi
return 0
}
check_xml_element_contains() {
if [ $# -lt 3 ]; then
log 2 "'check_xml_element_contains' requires data source, expected value, XML tree"
return 1
fi
if ! xml_val=$(get_element_text "$1" "${@:3}"); then
log 2 "error getting element text"
return 1
fi
if [[ "$xml_val" != *"$2"* ]]; then
log 2 "XML data mismatch, expected '$2', actual '$xml_val'"
return 1
fi
return 0
}
check_xml_error_contains() {
if [ "$#" -ne 3 ]; then
log 2 "'check_xml_code_error_contains' requires data source, expected error, string"
return 1
fi
if ! check_xml_element "$1" "$2" "Error" "Code"; then
log 2 "error checking xml error code"
return 1
fi
if ! check_xml_element_contains "$1" "$3" "Error" "Message"; then
log 2 "error checking xml element"
return 1
fi
return 0
}