mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-19 21:56:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: README updates, speed up some file integrity checks

Browse Source
This commit is contained in:
Luke McCrone
2026-06-23 20:37:14 -03:00
parent bf8b18839f
commit dba147380e
15 changed files with 155 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/system.yml
+1
View File
@@ -151,6 +151,7 @@ jobs:
AWS_REGION=$AWS_REGION
COVERAGE_LOG=$COVERAGE_LOG
TEMPLATE_MATRIX_FILE=$TEMPLATE_MATRIX_FILE
QUICK_COMPARE_SIZE=1048576
EOF
sudo chown tester:tester /home/tester/test-files/.env
sudo chmod 600 /home/tester/test-files/.env
tests/README.md
+2 -2
View File
@@ -45,8 +45,8 @@
openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365
```
10. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets, for older and static bucket tests, or prefixes, for newer and non-static bucket tests. If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`. To print options, run `tests/run.sh -h`. To run all tests (not recommended), run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
10. The program uses two environment values for the bucket names (or prefixes): `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME`. Originally, and for static and some older tests, these are the full bucket names. In later, non-static tests, these are prefixes, and bucket suffixes are auto-generated. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets, for older and static bucket tests, or prefixes, for newer and non-static bucket tests. If you want static buckets, i.e. you don't want them to be re-created each test, set `RECREATE_BUCKETS` to `false`.
11. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`. To print options, run `tests/run.sh -h`. To run all tests (not currently recommended due to long running time), run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
12. BATS tests can also be run directly with the format `VERSIYTGW_TEST_ENV=<env file> tests/<test file name>`, or for single tests, `VERSIYTGW_TEST_ENV=<env file> tests/<file name> -f <test name>`. Example: `VERSITYGW_TEST_ENV=tests/.env tests/test_rest_bucket.sh -f "REST - HeadBucket"`.
#### Tags
tests/drivers/file.sh
+25
View File
@@ -790,6 +790,31 @@ download_and_compare_file() {
return 0
}
check_file_integrity() {
if ! check_param_count_ge_le "original file, bucket, key, destination, chunk size (optional)" 4 5 $#; then
return 1
fi
local response compare_result=0
log 5 "quick compare size: '$QUICK_COMPARE_SIZE'"
if [ -n "$QUICK_COMPARE_SIZE" ]; then
log 5 "checking quick compare"
check_quick_compare "$1" "$2" "$3" || compare_result=$?
if [ "$compare_result" -eq 0 ]; then
return 0
elif [ "$compare_result" -eq 1 ] || [ "$compare_result" -eq 3 ]; then
return 1
fi
log 5 "skipping quick compare"
fi
log 5 "performing slow compare"
if ! download_and_compare_file "$1" "$2" "$3" "$4" "$5"; then
log 2 "error downloading and comparing file"
return 1
fi
return 0
}
# params: src, dst
# fail if error
copy_file_locally() {
tests/drivers/head_object/head_object_rest.sh
+87
View File
@@ -286,3 +286,90 @@ get_delete_marker_and_verify_405() {
fi
return 0
}
parse_checksum_and_file_size() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
local key value lowercase_key checksum_algorithm checksum checksum_type file_size
while IFS=$': \r' read -r key value; do
lowercase_key=${key,,}
if [[ "$lowercase_key" == "x-amz-checksum-type" ]]; then
checksum_type="$value"
elif [[ "$lowercase_key" == "x-amz-checksum-"* ]]; then
checksum_algorithm="${lowercase_key/x-amz-checksum-/}"
checksum="$value"
elif [[ "$lowercase_key" == "content-length" ]]; then
file_size="$value"
fi
done <<< "$(grep -aE '^.+: .+$' "$1")"
echo "${checksum_algorithm:-none} ${checksum:-none} ${checksum_type:-none} ${file_size:-none}"
return 0
}
get_checksum_and_file_size() {
if ! check_param_count_v2 "bucket name, object key" 2 $#; then
return 1
fi
local response
if ! response=$(send_rest_go_command_callback "200" "parse_checksum_and_file_size" "-method" "HEAD" "-bucketName" "$1" "-objectKey" "$2" "-signedParams" "x-amz-checksum-mode:ENABLED" 2>&1); then
log 2 "error sending HeadObject command and getting checksum and file size: $response"
return 1
fi
echo "$response"
return 0
}
# return 3 for error, 2 for skip, 1 for mismatch, 0 for match
check_quick_compare() {
if ! check_param_count_v2 "local file, bucket name, object key" 3 $#; then
return 1
fi
local response local_file_size checksum_algorithm remote_checksum checksum_type remote_key_size
if [ -z "$QUICK_COMPARE_SIZE" ]; then
log 5 "no QUICK_COMPARE_SIZE env param"
return 2
fi
if ! response=$(get_file_size "$1" 2>&1); then
log 2 "error getting file size: $response"
return 1
fi
local_file_size="$response"
if [ "$local_file_size" -le "$QUICK_COMPARE_SIZE" ]; then
log 5 "file size '$local_file_size' less than quick compare size '$QUICK_COMPARE_SIZE'"
return 2
fi
if ! response=$(get_checksum_and_file_size "$2" "$3" 2>&1); then
log 2 "error getting checksum and file size: $response"
return 3
fi
log 5 "RESPONSE: $response"
read -r checksum_algorithm remote_checksum checksum_type remote_key_size <<< "$response"
if [ "$remote_checksum" == "none" ] || [ "$checksum_algorithm" == "none" ] || [ "$checksum_type" != "FULL_OBJECT" ] || [ "$remote_key_size" == "none" ]; then
log 5 "skipping calcuation, no checksum or checksum algorithm, wrong checksum type, or key size missing"
return 2
fi
if [ "$local_file_size" -ne "$remote_key_size" ]; then
log 2 "file size mismatch ('$local_file_size' locally, '$remote_key_size' remotely)"
return 1
fi
if ! local_checksum=$(DATA_FILE="$1" CHECKSUM_TYPE="$checksum_algorithm" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
log 2 "error calculating checksum: $local_checksum"
return 3
fi
if [ "$remote_checksum" != "$local_checksum" ]; then
log 2 "checksum mismatch ('$local_checksum' locally, '$remote_checksum' remotely)"
return 1
fi
return 0
}
tests/env.sh
+3
View File
@@ -199,6 +199,9 @@ check_universal_vars() {
if [ -n "$COVERAGE_LOG" ]; then
export COVERAGE_LOG
fi
if [ -n "$QUICK_COMPARE_SIZE" ]; then
export QUICK_COMPARE_SIZE
fi
if ! check_aws_vars; then
log 1 "error checking AWS-related env vars"
tests/test_common.sh
+3 -3
View File
@@ -60,7 +60,7 @@ test_common_multipart_upload() {
fi
log 5 "file: $TEST_FILE_FOLDER/$bucket_file, bucket: $bucket_name"
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
assert_success
}
@@ -122,7 +122,7 @@ test_common_copy_object() {
run copy_object "$1" "$bucket_one/$object_name" "$bucket_two" "$object_name"
assert_success
fi
run download_and_compare_file "$TEST_FILE_FOLDER/$object_name" "$bucket_two" "$object_name" "$TEST_FILE_FOLDER/$object_name-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$object_name" "$bucket_two" "$object_name" "$TEST_FILE_FOLDER/$object_name-copy"
assert_success
}
@@ -176,7 +176,7 @@ test_common_put_object() {
assert_success
fi
run download_and_compare_file "$TEST_FILE_FOLDER/$2" "$bucket_name" "$2" "$TEST_FILE_FOLDER/${2}-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$2" "$bucket_name" "$2" "$TEST_FILE_FOLDER/${2}-copy"
assert_success
run delete_object "$1" "$bucket_name" "$2"
tests/test_rest_chunked.sh
+5 -5
View File
@@ -100,7 +100,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro
run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
assert_success
}
@@ -123,7 +123,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro
run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
assert_success
}
@@ -146,7 +146,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro
run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
assert_success
}
@@ -169,7 +169,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro
run chunked_upload_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
assert_success
}
@@ -299,7 +299,7 @@ source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_contro
run chunked_upload_trailer_different_chunk_size "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "sha256"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
assert_success
}
tests/test_rest_delete_object.sh
+1 -1
View File
@@ -37,7 +37,7 @@ source ./tests/setup.sh
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
assert_success
run delete_object "rest" "$bucket_name" "$test_file"
tests/test_rest_delete_objects.sh
+3 -3
View File
@@ -67,7 +67,7 @@ source ./tests/setup.sh
"-contentMD5" "-objectsToDelete" "key=$incorrect_file_name" "--" "$incorrect_file_name" "DeleteResult" "Deleted" "Key"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy"
assert_success
}
@@ -87,7 +87,7 @@ source ./tests/setup.sh
"-contentMD5" "-objectsToDelete" "key=$file_name;eTag=abc" "--" "$file_name"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy"
assert_success
}
@@ -129,7 +129,7 @@ source ./tests/setup.sh
"-contentMD5" "-objectsToDelete" "key=$file_name;versionId=$incorrect_version" "--" "$file_name" "$incorrect_version"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name" "$TEST_FILE_FOLDER/${file_name}-copy"
assert_success
}
tests/test_rest_head_object.sh
+17
View File
@@ -133,3 +133,20 @@ source ./tests/drivers/string.sh
assert_success
done
}
@test "REST - quick compare" {
run setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 5
assert_success
read -r bucket_name file_name <<< "$output"
run put_object_rest "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
assert_success
export QUICK_COMPARE_SIZE=1000000
run check_quick_compare "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
assert_success
export QUICK_COMPARE_SIZE=10000000
run check_quick_compare "$TEST_FILE_FOLDER/$file_name" "$bucket_name" "$file_name"
assert_failure 2
}
tests/test_rest_multipart.sh
+2 -2
View File
@@ -57,7 +57,7 @@ source ./tests/drivers/upload_part/upload_part_rest.sh
"$part_one" "$part_two" "$part_three" "$part_four"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$large_test_file" "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$large_test_file" "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file-copy"
assert_success
}
@@ -97,7 +97,7 @@ source ./tests/drivers/upload_part/upload_part_rest.sh
run create_upload_part_copy_rest "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$large_test_file" "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$large_test_file" "$bucket_name" "$large_test_file" "$TEST_FILE_FOLDER/$large_test_file-copy"
assert_success
}
tests/test_rest_put_object.sh
+1 -1
View File
@@ -572,6 +572,6 @@ export RUN_USERS=true
run send_openssl_go_command "200" "-method" "PUT" "-bucketName" "$bucket_name" "-objectKey" "$test_file" "-payload" "$payload_content"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/${test_file}_downloaded"
run check_file_integrity "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/${test_file}_downloaded"
assert_success
}
tests/test_s3api_multipart.sh
+3 -3
View File
@@ -59,7 +59,7 @@ export RUN_USERS=true
run multipart_upload "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
assert_success
}
@@ -104,7 +104,7 @@ export RUN_USERS=true
run get_and_check_legal_hold "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "OFF"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
assert_success
}
@@ -117,7 +117,7 @@ export RUN_USERS=true
run multipart_upload_from_bucket "$bucket_name" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "${bucket_file}-copy" "$TEST_FILE_FOLDER/$bucket_file-copy-two"
run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$bucket_name" "${bucket_file}-copy" "$TEST_FILE_FOLDER/$bucket_file-copy-two"
assert_success
}
tests/test_s3api_policy_object.sh
+1 -1
View File
@@ -332,7 +332,7 @@ test_s3api_policy_put_wildcard() {
run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_folder/$test_file-copy" "$username" "$password"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
run check_file_integrity "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
assert_success
}
tests/test_s3api_root_inner.sh
+1 -1
View File
@@ -89,7 +89,7 @@ test_put_object_s3api_root() {
run copy_object "s3api" "$bucket_one/$bucket_file" "$bucket_two" "$bucket_file"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$bucket_file" "$bucket_two" "$bucket_file" "$TEST_FILE_FOLDER/${bucket_file}_copy"
run check_file_integrity "$TEST_FILE_FOLDER/$bucket_file" "$bucket_two" "$bucket_file" "$TEST_FILE_FOLDER/${bucket_file}_copy"
assert_success
}