diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index b519951b..16b7263c 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -151,6 +151,7 @@ jobs: AWS_REGION=$AWS_REGION COVERAGE_LOG=$COVERAGE_LOG TEMPLATE_MATRIX_FILE=$TEMPLATE_MATRIX_FILE + QUICK_COMPARE_SIZE=1048576 EOF sudo chown tester:tester /home/tester/test-files/.env sudo chmod 600 /home/tester/test-files/.env diff --git a/tests/README.md b/tests/README.md index a2d122e3..eb293696 100644 --- a/tests/README.md +++ b/tests/README.md @@ -45,8 +45,8 @@ openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048 openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 ``` -10. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets, for older and static bucket tests, or prefixes, for newer and non-static bucket tests. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`. -11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV= tests/run.sh `. To print options, run `tests/run.sh -h`. To run all tests (not recommended), run `VERSITYGW_TEST_ENV= tests/run_all.sh`. +10. The program uses two environment values for the bucket names (or prefixes): `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME`. Originally, and for static and some older tests, these are the full bucket names. In later, non-static tests, these are prefixes, and bucket suffixes are auto-generated. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets, for older and static bucket tests, or prefixes, for newer and non-static bucket tests. If you want static buckets, i.e. you don't want them to be re-created each test, set `RECREATE_BUCKETS` to `false`. +11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV= tests/run.sh `. To print options, run `tests/run.sh -h`. To run all tests (not currently recommended due to long running time), run `VERSITYGW_TEST_ENV= tests/run_all.sh`. 12. BATS tests can also be run directly with the format `VERSIYTGW_TEST_ENV= tests/`, or for single tests, `VERSIYTGW_TEST_ENV= tests/ -f `. Example: `VERSITYGW_TEST_ENV=tests/.env tests/test_rest_bucket.sh -f "REST - HeadBucket"`. #### Tags diff --git a/tests/drivers/file.sh b/tests/drivers/file.sh index 83aed0ab..975d81e0 100644 --- a/tests/drivers/file.sh +++ b/tests/drivers/file.sh @@ -790,6 +790,31 @@ download_and_compare_file() { return 0 } +check_file_integrity() { + if ! check_param_count_ge_le "original file, bucket, key, destination, chunk size (optional)" 4 5 $#; then + return 1 + fi + local response compare_result=0 + + log 5 "quick compare size: '$QUICK_COMPARE_SIZE'" + if [ -n "$QUICK_COMPARE_SIZE" ]; then + log 5 "checking quick compare" + check_quick_compare "$1" "$2" "$3" || compare_result=$? + if [ "$compare_result" -eq 0 ]; then + return 0 + elif [ "$compare_result" -eq 1 ] || [ "$compare_result" -eq 3 ]; then + return 1 + fi + log 5 "skipping quick compare" + fi + log 5 "performing slow compare" + if ! download_and_compare_file "$1" "$2" "$3" "$4" "$5"; then + log 2 "error downloading and comparing file" + return 1 + fi + return 0 +} + # params: src, dst # fail if error copy_file_locally() { diff --git a/tests/drivers/head_object/head_object_rest.sh b/tests/drivers/head_object/head_object_rest.sh index 369f1015..5722e78a 100644 --- a/tests/drivers/head_object/head_object_rest.sh +++ b/tests/drivers/head_object/head_object_rest.sh @@ -286,3 +286,90 @@ get_delete_marker_and_verify_405() { fi return 0 } + +parse_checksum_and_file_size() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + local key value lowercase_key checksum_algorithm checksum checksum_type file_size + + while IFS=$': \r' read -r key value; do + lowercase_key=${key,,} + if [[ "$lowercase_key" == "x-amz-checksum-type" ]]; then + checksum_type="$value" + elif [[ "$lowercase_key" == "x-amz-checksum-"* ]]; then + checksum_algorithm="${lowercase_key/x-amz-checksum-/}" + checksum="$value" + elif [[ "$lowercase_key" == "content-length" ]]; then + file_size="$value" + fi + done <<< "$(grep -aE '^.+: .+$' "$1")" + + echo "${checksum_algorithm:-none} ${checksum:-none} ${checksum_type:-none} ${file_size:-none}" + return 0 +} + +get_checksum_and_file_size() { + if ! check_param_count_v2 "bucket name, object key" 2 $#; then + return 1 + fi + local response + + if ! response=$(send_rest_go_command_callback "200" "parse_checksum_and_file_size" "-method" "HEAD" "-bucketName" "$1" "-objectKey" "$2" "-signedParams" "x-amz-checksum-mode:ENABLED" 2>&1); then + log 2 "error sending HeadObject command and getting checksum and file size: $response" + return 1 + fi + echo "$response" + return 0 +} + +# return 3 for error, 2 for skip, 1 for mismatch, 0 for match +check_quick_compare() { + if ! check_param_count_v2 "local file, bucket name, object key" 3 $#; then + return 1 + fi + local response local_file_size checksum_algorithm remote_checksum checksum_type remote_key_size + + if [ -z "$QUICK_COMPARE_SIZE" ]; then + log 5 "no QUICK_COMPARE_SIZE env param" + return 2 + fi + + if ! response=$(get_file_size "$1" 2>&1); then + log 2 "error getting file size: $response" + return 1 + fi + local_file_size="$response" + + if [ "$local_file_size" -le "$QUICK_COMPARE_SIZE" ]; then + log 5 "file size '$local_file_size' less than quick compare size '$QUICK_COMPARE_SIZE'" + return 2 + fi + + if ! response=$(get_checksum_and_file_size "$2" "$3" 2>&1); then + log 2 "error getting checksum and file size: $response" + return 3 + fi + log 5 "RESPONSE: $response" + read -r checksum_algorithm remote_checksum checksum_type remote_key_size <<< "$response" + + if [ "$remote_checksum" == "none" ] || [ "$checksum_algorithm" == "none" ] || [ "$checksum_type" != "FULL_OBJECT" ] || [ "$remote_key_size" == "none" ]; then + log 5 "skipping calcuation, no checksum or checksum algorithm, wrong checksum type, or key size missing" + return 2 + fi + + if [ "$local_file_size" -ne "$remote_key_size" ]; then + log 2 "file size mismatch ('$local_file_size' locally, '$remote_key_size' remotely)" + return 1 + fi + + if ! local_checksum=$(DATA_FILE="$1" CHECKSUM_TYPE="$checksum_algorithm" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then + log 2 "error calculating checksum: $local_checksum" + return 3 + fi + if [ "$remote_checksum" != "$local_checksum" ]; then + log 2 "checksum mismatch ('$local_checksum' locally, '$remote_checksum' remotely)" + return 1 + fi + return 0 +} diff --git a/tests/env.sh b/tests/env.sh index f3b42197..0b0959cc 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -199,6 +199,9 @@ check_universal_vars() { if [ -n "$COVERAGE_LOG" ]; then export COVERAGE_LOG fi + if [ -n "$QUICK_COMPARE_SIZE" ]; then + export QUICK_COMPARE_SIZE + fi if ! check_aws_vars; then log 1 "error checking AWS-related env vars" diff --git a/tests/test_common.sh b/tests/test_common.sh index c582e1e9..c9413c3e 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -60,7 +60,7 @@ test_common_multipart_upload() { fi log 5 "file: $TEST_FILE_FOLDER/$bucket_file, bucket: $bucket_name" - run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" assert_success } @@ -122,7 +122,7 @@ test_common_copy_object() { run copy_object "$1" "$bucket_one/$object_name" "$bucket_two" "$object_name" assert_success fi - run download_and_compare_file "$TEST_FILE_FOLDER/$object_name" "$bucket_two" "$object_name" "$TEST_FILE_FOLDER/$object_name-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$object_name" "$bucket_two" "$object_name" "$TEST_FILE_FOLDER/$object_name-copy" assert_success } @@ -176,7 +176,7 @@ test_common_put_object() { assert_success fi - run download_and_compare_file "$TEST_FILE_FOLDER/$2" "$bucket_name" "$2" "$TEST_FILE_FOLDER/${2}-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$2" "$bucket_name" "$2" "$TEST_FILE_FOLDER/${2}-copy" assert_success run delete_object "$1" "$bucket_name" "$2" diff --git a/tests/test_rest_chunked.sh b/tests/test_rest_chunked.sh index 7f7f8ee0..eccf947c 100755 --- a/tests/test_rest_chunked.sh +++ b/tests/test_rest_chunked.sh @@ -100,7 +100,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" assert_success } @@ -123,7 +123,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" assert_success } @@ -146,7 +146,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" assert_success } @@ -169,7 +169,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" assert_success } @@ -299,7 +299,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro run chunked_upload_trailer_different_chunk_size "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "sha256" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" assert_success } diff --git a/tests/test_rest_delete_object.sh b/tests/test_rest_delete_object.sh index 7052f719..825be56d 100755 --- a/tests/test_rest_delete_object.sh +++ b/tests/test_rest_delete_object.sh @@ -37,7 +37,7 @@ source ./tests/setup.sh run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" assert_success run delete_object "rest" "$bucket_name" "$test_file" diff --git a/tests/test_rest_delete_objects.sh b/tests/test_rest_delete_objects.sh index 03272b71..eb0558e8 100755 --- a/tests/test_rest_delete_objects.sh +++ b/tests/test_rest_delete_objects.sh @@ -67,7 +67,7 @@ source ./tests/setup.sh "-contentMD5" "-objectsToDelete" "key=$incorrect_file_name" "--" "$incorrect_file_name" "DeleteResult" "Deleted" "Key" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" assert_success } @@ -87,7 +87,7 @@ source ./tests/setup.sh "-contentMD5" "-objectsToDelete" "key=$file_name;eTag=abc" "--" "$file_name" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" assert_success } @@ -129,7 +129,7 @@ source ./tests/setup.sh "-contentMD5" "-objectsToDelete" "key=$file_name;versionId=$incorrect_version" "--" "$file_name" "$incorrect_version" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy" assert_success } diff --git a/tests/test_rest_head_object.sh b/tests/test_rest_head_object.sh index 24c12605..5b4bb1a5 100755 --- a/tests/test_rest_head_object.sh +++ b/tests/test_rest_head_object.sh @@ -133,3 +133,20 @@ source ./tests/drivers/string.sh assert_success done } + +@test "REST - quick compare" { + run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 5 + assert_success + read -r bucket_name file_name <<< "$output" + + run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + export QUICK_COMPARE_SIZE=1000000 + run check_quick_compare "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_success + + export QUICK_COMPARE_SIZE=10000000 + run check_quick_compare "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" + assert_failure 2 +} diff --git a/tests/test_rest_multipart.sh b/tests/test_rest_multipart.sh index 67de6b5c..19f90c08 100755 --- a/tests/test_rest_multipart.sh +++ b/tests/test_rest_multipart.sh @@ -57,7 +57,7 @@ source ./tests/drivers/upload_part/upload_part_rest.sh "$part_one" "$part_two" "$part_three" "$part_four" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$large_test_file" "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$large_test_file" "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file-copy" assert_success } @@ -97,7 +97,7 @@ source ./tests/drivers/upload_part/upload_part_rest.sh run create_upload_part_copy_rest "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$large_test_file" "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$large_test_file" "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file-copy" assert_success } diff --git a/tests/test_rest_put_object.sh b/tests/test_rest_put_object.sh index c0d84145..37cf8a86 100755 --- a/tests/test_rest_put_object.sh +++ b/tests/test_rest_put_object.sh @@ -572,6 +572,6 @@ export RUN_USERS=true run send_openssl_go_command "200" "-method" "PUT" "-bucketName" "$bucket_name" "-objectKey" "$test_file" "-payload" "$payload_content" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/${test_file}_downloaded" + run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/${test_file}_downloaded" assert_success } diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index 288d7f77..d79a744c 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -59,7 +59,7 @@ export RUN_USERS=true run multipart_upload "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" assert_success } @@ -104,7 +104,7 @@ export RUN_USERS=true run get_and_check_legal_hold "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "OFF" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" assert_success } @@ -117,7 +117,7 @@ export RUN_USERS=true run multipart_upload_from_bucket "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4 assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "${bucket_file}-copy" "$TEST_FILE_FOLDER/$bucket_file-copy-two" + run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "${bucket_file}-copy" "$TEST_FILE_FOLDER/$bucket_file-copy-two" assert_success } diff --git a/tests/test_s3api_policy_object.sh b/tests/test_s3api_policy_object.sh index 793eea95..a1ee402c 100644 --- a/tests/test_s3api_policy_object.sh +++ b/tests/test_s3api_policy_object.sh @@ -332,7 +332,7 @@ test_s3api_policy_put_wildcard() { run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_folder/$test_file-copy" "$username" "$password" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$TEST_FILE_FOLDER/$test_file-copy" + run check_file_integrity "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$TEST_FILE_FOLDER/$test_file-copy" assert_success } diff --git a/tests/test_s3api_root_inner.sh b/tests/test_s3api_root_inner.sh index 58190c09..722a0e4a 100755 --- a/tests/test_s3api_root_inner.sh +++ b/tests/test_s3api_root_inner.sh @@ -89,7 +89,7 @@ test_put_object_s3api_root() { run copy_object "s3api" "$bucket_one/$bucket_file" "$bucket_two" "$bucket_file" assert_success - run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_two" "$bucket_file" "$TEST_FILE_FOLDER/${bucket_file}_copy" + run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$bucket_two" "$bucket_file" "$TEST_FILE_FOLDER/${bucket_file}_copy" assert_success }