test: crc32c checksum

This commit is contained in:
Luke McCrone
2025-03-01 22:58:18 -03:00
parent 4c5cd918d8
commit 485f2abbaa
9 changed files with 242 additions and 145 deletions

View File

@@ -58,29 +58,7 @@ source_config_file() {
fi
}
check_universal_vars() {
if [[ $BYPASS_ENV_FILE != "true" ]]; then
source_config_file
fi
if [ -n "$COMMAND_LOG" ]; then
init_command_log
fi
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
# shellcheck source=./tests/.secrets
source "$SECRETS_FILE"
else
log 3 "Warning: no secrets file found"
fi
if [[ -n "$LOG_LEVEL" ]]; then
if [[ $LOG_LEVEL -lt 2 ]]; then
log 1 "log level must be 2 or greater"
exit 1
fi
export LOG_LEVEL_INT=$LOG_LEVEL
fi
check_aws_vars() {
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
log 1 "AWS_ACCESS_KEY_ID missing"
exit 1
@@ -103,10 +81,14 @@ check_universal_vars() {
exit 1
fi
fi
if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
fail "RUN_VERSITYGW must be 'true' or 'false'"
# exporting these since they're needed for subshells
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
if [ -n "$AWS_CANONICAL_ID" ]; then
export AWS_CANONICAL_ID
fi
}
check_bucket_vars() {
if [ -z "$BUCKET_ONE_NAME" ]; then
log 1 "BUCKET_ONE_NAME missing"
exit 1
@@ -129,24 +111,50 @@ check_universal_vars() {
fi
if [ "$RECREATE_BUCKETS" == "false" ] && [ "$DELETE_BUCKETS_AFTER_TEST" == "true" ]; then
log 1 "cannot set DELETE_BUCKETS_AFTER_TEST to 'true' if RECREATE_BUCKETS is 'false'"
return 1
exit 1
fi
}
check_universal_vars() {
if [[ $BYPASS_ENV_FILE != "true" ]]; then
source_config_file
fi
if [ -n "$COMMAND_LOG" ]; then
init_command_log
fi
if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
# shellcheck source=./tests/.secrets
source "$SECRETS_FILE"
else
log 3 "Warning: no secrets file found"
fi
if [[ -n "$LOG_LEVEL" ]]; then
if [[ $LOG_LEVEL -lt 2 ]]; then
log 1 "log level must be 2 or greater"
exit 1
fi
export LOG_LEVEL_INT=$LOG_LEVEL
fi
check_aws_vars
if [ "$RUN_VERSITYGW" != "true" ] && [ "$RUN_VERSITYGW" != "false" ]; then
log 1 "RUN_VERSITYGW must be 'true' or 'false'"
exit 1
fi
check_bucket_vars
if [ -z "$TEST_FILE_FOLDER" ]; then
log 1 "TEST_FILE_FOLDER missing"
exit 1
fi
if [ ! -d "$TEST_FILE_FOLDER" ]; then
if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then
log 2 "error creating test folder: $error"
log 1 "error creating test folder: $error"
exit 1
fi
fi
# exporting these since they're needed for subshells
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
if [ -n "$AWS_CANONICAL_ID" ]; then
log 5 "canonical ID: $AWS_CANONICAL_ID"
export AWS_CANONICAL_ID
fi
}
delete_command_log() {

View File

@@ -25,7 +25,7 @@ import sys
from awscrt import checksums
with open(sys.argv[1], 'rb') as f:
print(checksums.crc64nvme(f.read()))" "$DATA_FILE" 2>&1); then
print(checksums.${CHECKSUM_TYPE}(f.read()))" "$DATA_FILE" 2>&1); then
log_rest 2 "error calculating checksum: $checksum_decimal"
exit 1
fi
@@ -34,5 +34,11 @@ if ! deactivate 1>/dev/null; then
log_rest 2 "error deactivating virtual environment"
exit 1
fi
checksum_hash=$(printf "%016x" "$checksum_decimal" | xxd -r -p | base64)
if [ "$CHECKSUM_TYPE" == "crc64nvme" ]; then
hex_format="%016x"
else
hex_format="%08x"
fi
# shellcheck disable=SC2059
checksum_hash=$(printf "$hex_format" "$checksum_decimal" | xxd -r -p | base64)
echo "$checksum_hash"

View File

@@ -46,19 +46,30 @@ if [ "$checksum_type" == "sha256" ]; then
checksum_hash="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
fi
cr_data+=("x-amz-checksum-sha256:$checksum_hash")
elif [ "$checksum_type" == "sha1" ]; then
if [ -z "$checksum_hash" ]; then
checksum_hash="$(sha1sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
fi
cr_data+=("x-amz-checksum-sha1:$checksum_hash")
elif [ "$checksum_type" == "crc32" ]; then
if [ -z "$checksum_hash" ]; then
checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
fi
cr_data+=("x-amz-checksum-crc32:$checksum_hash")
elif [ "$checksum_type" == "crc64nvme" ]; then
if [ -z "$checksum_hash" ]; then
if ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
exit 1
fi
if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc64nvme" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
exit 1
fi
cr_data+=("x-amz-checksum-crc64nvme:$checksum_hash")
elif [ "$checksum_type" == "crc32c" ]; then
if [ -z "$checksum_hash" ] && ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc32c" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
log_rest 2 "error calculating crc32c checksum: $checksum_hash"
exit 1
fi
cr_data+=("x-amz-checksum-crc32c:$checksum_hash")
elif [ "$checksum_type" != "" ]; then
cr_data+=("x-amz-checksum-$checksum_type:$checksum_hash")
fi
cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
build_canonical_request "${cr_data[@]}"

View File

@@ -375,7 +375,7 @@ test_file="test_file"
@test "REST - head object" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1018"
skip "https://github.com/versity/versitygw/issues/1114"
fi
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
@@ -449,9 +449,6 @@ test_file="test_file"
}
@test "REST - HeadObject does not return 405 with versioning, after file deleted" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1029"
fi
if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
skip
fi

View File

@@ -23,6 +23,32 @@ source ./tests/util/util_setup.sh
test_file="test_file"
@test "REST - invalid checksum type" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1104"
fi
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run check_invalid_checksum_type "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success
}
@test "REST - sha256 checksum - invalid" {
run check_checksum_rest_invalid "sha256"
assert_success
}
@test "REST - sha256 checksum - incorrect" {
run check_checksum_rest_incorrect "sha256"
assert_success
}
@test "REST - sha256 checksum - correct" {
run add_correct_checksum "sha256"
assert_success
}
@test "REST - HeadObject returns x-amz-checksum-sha256" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
@@ -34,23 +60,22 @@ test_file="test_file"
assert_success
}
@test "REST - PutObject rejects invalid sha256 checksum" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object_rest_sha256_invalid "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@test "REST - crc32 checksum - invalid" {
run check_checksum_rest_invalid "crc32c"
assert_success
}
@test "REST - PutObject rejects incorrect sha256 checksum" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object_rest_sha256_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@test "REST - crc32 checksum - incorrect" {
run check_checksum_rest_incorrect "crc32"
assert_success
}
@test "REST - crc32 checksum - correct" {
run add_correct_checksum "crc32"
assert_success
}
@test "REST - crc32 checksum - HeadObject" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
@@ -61,26 +86,47 @@ test_file="test_file"
assert_success
}
@test "REST - crc32 checksum - incorrect" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object_rest_crc32_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success
}
@test "REST - crc64nvme checksum - correct" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc64nvme"
@test "REST - crc64nvme checksum - invalid" {
run check_checksum_rest_invalid "crc64nvme"
assert_success
}
@test "REST - crc64nvme checksum - incorrect" {
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object_rest_crc64nvme_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
run check_checksum_rest_incorrect "crc64nvme"
assert_success
}
@test "REST - crc64nvme checksum - correct" {
run add_correct_checksum "sha256"
assert_success
}
@test "REST - crc32c checksum - invalid" {
run check_checksum_rest_invalid "crc32c"
assert_success
}
@test "REST - crc32c checksum - incorrect" {
run check_checksum_rest_incorrect "crc32c"
assert_success
}
@test "REST - crc32c checksum - correct" {
run add_correct_checksum "crc32c"
assert_success
}
@test "REST - sha1 checksum - invalid" {
run check_checksum_rest_invalid "sha1"
assert_success
}
@test "REST - sha1 checksum - incorrect" {
run check_checksum_rest_incorrect "sha1"
assert_success
}
@test "REST - sha1 checksum - correct" {
run add_correct_checksum "sha1"
assert_success
}

View File

@@ -72,9 +72,6 @@ export RUN_USERS=true
# delete-objects
@test "test_delete_objects" {
if [ "$RECREATE_BUCKETS" == "false" ]; then
skip "https://github.com/versity/versitygw/issues/1029"
fi
test_delete_objects_s3api_root
}
@@ -128,16 +125,10 @@ export RUN_USERS=true
# test adding and removing an object on versitygw
@test "test_put_object_with_data" {
if [ "$RECREATE_BUCKETS" == "false" ]; then
skip "https://github.com/versity/versitygw/issues/1029"
fi
test_common_put_object_with_data "s3api"
}
@test "test_put_object_no_data" {
if [ "$RECREATE_BUCKETS" == "false" ]; then
skip "https://github.com/versity/versitygw/issues/1029"
fi
test_common_put_object_no_data "s3api"
}

View File

@@ -166,13 +166,3 @@ check_checksum_rest_crc32() {
fi
return 0
}
crc64_file() {
source ./env/bin/activate
python3 -m pip install awscrt
if ! checksum=$(python3 -c "import sys;from awscrt import checksums;crc64nvme_value = checksums.crc64nvme(sys.argv[1]);print(crc64nvme_value)" "$1"); then
return 1
fi
log 5 "checksum: $checksum"
deactivate
}

View File

@@ -14,6 +14,33 @@
# specific language governing permissions and limitations
# under the License.
multipart_upload_s3api_complete_from_bucket() {
if [ $# -ne 3 ]; then
log 2 "'multipart_upload_s3api_complete_from_bucket' requires bucket, copy source, part count"
return 1
fi
parts="["
for ((i = 1; i <= $3; i++)); do
# shellcheck disable=SC2154
if ! upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i"; then
log 2 "error uploading part $i"
return 1
fi
# shellcheck disable=SC2154
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
if [[ $i -ne $3 ]]; then
parts+=","
fi
done
parts+="]"
if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}' 2>&1); then
log 2 "Error completing upload: $error"
return 1
fi
return 0
}
multipart_upload_from_bucket() {
if [ $# -ne 4 ]; then
log 2 "multipart upload from bucket command missing bucket, copy source, key, and/or part count"
@@ -38,23 +65,8 @@ multipart_upload_from_bucket() {
return 1
fi
parts="["
for ((i = 1; i <= $4; i++)); do
# shellcheck disable=SC2154
if ! upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i"; then
log 2 "error uploading part $i"
return 1
fi
# shellcheck disable=SC2154
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
if [[ $i -ne $4 ]]; then
parts+=","
fi
done
parts+="]"
if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}' 2>&1); then
log 2 "Error completing upload: $error"
if ! multipart_upload_s3api_complete_from_bucket "$1" "$2" "$4"; then
log 2 " error completing multipart upload from bucket"
return 1
fi
return 0
@@ -181,8 +193,6 @@ multipart_upload_with_params() {
return 0
}
run_and_verify_multipart_upload_with_valid_range() {
if [ $# -ne 3 ]; then
log 2 "'run_and_verify_multipart_upload_with_valid_range' requires bucket, key, 5MB file"

View File

@@ -307,7 +307,7 @@ check_checksum_invalid_or_incorrect() {
return 1
fi
if [ "$result" != "400" ]; then
log 2 "expected response code of '400', was '$result' (response: $(cat "$TEST_FILE_FOLDER/result.txt")"
log 2 "expected response code of '400', was '$result' (response: $(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
if ! check_xml_element "$TEST_FILE_FOLDER/result.txt" "$6" "Error" "Message"; then
@@ -331,40 +331,87 @@ put_object_rest_checksum() {
log 2 "expected response code of '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
log 5 "result: $(cat "$TEST_FILE_FOLDER/result.txt")"
return 0
}
put_object_rest_sha256_invalid() {
if [ $# -ne 3 ]; then
log 2 "'put_object_rest_sha256_invalid' requires data file, bucket name, key"
check_checksum_rest_invalid() {
if [ $# -ne 1 ]; then
log 2 "'put_object_rest_sha256_invalid' requires checksum type"
return 1
fi
if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256" "dummy" "Value for x-amz-checksum-sha256 header is invalid."; then
test_file="test_file"
if ! setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"; then
log 2 "error setting up bucket and file"
return 1
fi
if ! check_checksum_invalid_or_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$1" "dummy" "Value for x-amz-checksum-$1 header is invalid."; then
log 2 "error checking checksum"
return 1
fi
return 0
}
put_object_rest_sha256_incorrect() {
if [ $# -ne 3 ]; then
log 2 "'put_object_rest_sha256_invalid' requires data file, bucket name, key"
check_checksum_rest_incorrect() {
if [ $# -ne 1 ]; then
log 2 "'check_checksum_rest_incorrect' requires checksum type"
return 1
fi
test_file="test_file"
if ! setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"; then
log 2 "error setting up bucket and file"
return 1
fi
if [ "$DIRECT" == "true" ]; then
error_message="The SHA256 you specified did not match the calculated checksum."
error_cs_str="$(echo "$1" | tr '[:lower:]' '[:upper:]')"
else
error_message="The sha256 you specified did not match the calculated checksum."
error_cs_str="$1"
fi
incorrect_checksum="$(echo -n "dummy" | sha256sum | awk '{print $1}' | xxd -r -p | base64)"
if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256" "$incorrect_checksum" "$error_message"; then
error_message="The $error_cs_str you specified did not match the calculated checksum."
if ! calculate_incorrect_checksum "$1" "$(cat "$TEST_FILE_FOLDER/$test_file")"; then
log 2 "error calculating incorrect checksum"
return 1
fi
if ! check_checksum_invalid_or_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$1" "$incorrect_checksum" "$error_message"; then
log 2 "error checking checksum"
return 1
fi
return 0
}
calculate_incorrect_checksum() {
if [ $# -ne 2 ]; then
log 2 "'calculate_incorrect_checksum' requires checksum type, data"
return 1
fi
case "$1" in
"sha1")
incorrect_checksum="$(echo -n "$2"a | sha1sum | awk '{print $1}' | xxd -r -p | base64)"
;;
"sha256")
incorrect_checksum="$(echo -n "$2"a | sha256sum | awk '{print $1}' | xxd -r -p | base64)"
;;
"crc32")
incorrect_checksum="$(echo -n "$2"a | gzip -c -1 | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
;;
"crc32c")
if ! incorrect_checksum=$(DATA_FILE=<(echo -n "$2"a) TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc32c" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
log 2 "error calculating checksum: $incorrect_checksum"
return 1
fi
;;
"crc64nvme")
if ! incorrect_checksum=$(DATA_FILE=<(echo -n "$2"a) TEST_FILE_FOLDER="$TEST_FILE_FOLDER" CHECKSUM_TYPE="crc64nvme" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
log 2 "error calculating checksum: $incorrect_checksum"
return 1
fi
;;
*)
log 2 "invalid checksum type: $1"
return 1
esac
return 0
}
put_object_rest_chunked_payload_type_without_content_length() {
if [ $# -ne 3 ]; then
log 2 "'put_object_rest_diff_payload_type' requires data file, bucket name, key"
@@ -381,41 +428,32 @@ put_object_rest_chunked_payload_type_without_content_length() {
return 0
}
put_object_rest_crc32_incorrect() {
if [ $# -ne 3 ]; then
log 2 "'put_object_rest_crc32_incorrect' requires data file, bucket name, key"
add_correct_checksum() {
if [ $# -ne 1 ]; then
log 2 "'add_correct_checksum' requires checksum type"
return 1
fi
if [ "$DIRECT" == "true" ]; then
error_message="The CRC32 you specified did not match the calculated checksum."
else
error_message="The crc32 you specified did not match the calculated checksum."
test_file="test_file"
if ! setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"; then
log 2 "error setting up bucket and file"
return 1
fi
incorrect_checksum="$(echo -n "dummy" | gzip -c -1 | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "crc32" "$incorrect_checksum" "$error_message"; then
log 2 "error checking checksum"
if ! put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$1"; then
log 2 "error adding file with checksum to s3"
return 1
fi
return 0
}
put_object_rest_crc64nvme_incorrect() {
check_invalid_checksum_type() {
if [ $# -ne 3 ]; then
log 2 "'put_object_rest_crc64nvme_incorrect' requires data file, bucket name, key"
log 2 "'check_invalid_checksum_type' requires data file, bucket name, file"
return 1
fi
if [ "$DIRECT" == "true" ]; then
error_message="The CRC64NVME you specified did not match the calculated checksum."
else
error_message="The crc64nvme you specified did not match the calculated checksum."
fi
if ! incorrect_checksum=$(DATA_FILE=<(echo -n "dummy") TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
log 2 "error calculating checksum: $incorrect_checksum"
return 1
fi
if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "crc64nvme" "$incorrect_checksum" "$error_message"; then
error_message='The algorithm type you specified in x-amz-checksum- header is invalid.'
if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256a" "dummy" "$error_message"; then
log 2 "error checking checksum"
return 1
fi
return 0
}