mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-01-05 03:24:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: multipart upload checksum tests

Browse Source
This commit is contained in:
Luke McCrone
2025-05-10 20:41:37 -03:00
parent 3ec6e634c3
commit 28cb97329e
49 changed files with 1242 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
tests/commands/complete_multipart_upload.sh
View File

@@ -28,4 +28,66 @@ complete_multipart_upload() {
fi
log 5 "complete multipart upload error: $error"
return 0
}
complete_multipart_upload_rest() {
if ! check_param_count_v2 "bucket, key, upload ID, parts payload" 4 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" PARTS="$4" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then
log 2 "error completing multipart upload: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
return 1
fi
}
complete_multipart_upload_rest_nonexistent_param() {
if ! check_param_count_v2 "bucket, key, upload ID, parts payload" 4 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" PARTS="$4" ALGORITHM_PARAMETER="true" CHECKSUM_ALGORITHM="crc32c" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh 2>&1); then
log 2 "error completing multipart upload: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
return 1
fi
}
complete_multipart_upload_rest_incorrect_checksum() {
if ! check_param_count_v2 "bucket, key, upload ID, parts payload, type, algorithm, correct hash" 7 $#; then
return 1
fi
checksum="$7"
if [ "${checksum:0:1}" == "a" ]; then
checksum="b${checksum:1}"
else
checksum="a${checksum:1}"
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" PARTS="$4" CHECKSUM_TYPE="$5" CHECKSUM_ALGORITHM="$6" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh 2>&1); then
log 2 "error completing multipart upload: $result"
return 1
fi
if ! check_rest_expected_error "$result" "$TEST_FILE_FOLDER/result.txt" 400 "BadDigest" "did not match"; then
log 2 "expected '400', was $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
return 1
fi
}
complete_multipart_upload_rest_invalid_checksum() {
if ! check_param_count_v2 "bucket, key, upload ID, parts payload, type, algorithm, correct hash" 7 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" PARTS="$4" CHECKSUM_TYPE="$5" CHECKSUM_ALGORITHM="$6" CHECKSUM_HASH="$7" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh 2>&1); then
log 2 "error completing multipart upload: $result"
return 1
fi
if ! check_rest_expected_error "$result" "$TEST_FILE_FOLDER/result.txt" 400 "InvalidRequest" "header is invalid"; then
log 2 "expected '400', was $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
return 1
fi
}

158
tests/commands/create_multipart_upload.sh
View File

@@ -14,13 +14,63 @@
# specific language governing permissions and limitations
# under the License.
# initialize a multipart upload
# params: bucket, key
# return 0 for success, 1 for failure
create_multipart_upload() {
create_multipart_upload_rest() {
if ! check_param_count_v2 "bucket name, key" 2 $#; then
return 1
fi
if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" COMMAND_LOG=$COMMAND_LOG ./tests/rest_scripts/create_multipart_upload.sh); then
log 2 "error creating multipart upload: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/output.txt")"
return 1
fi
if ! upload_id=$(get_element_text "$TEST_FILE_FOLDER/output.txt" "InitiateMultipartUploadResult" "UploadId"); then
log 2 "error getting upload ID: $upload_id"
return 1
fi
echo "$upload_id"
return 0
}
create_multipart_upload_rest_with_checksum_type_and_algorithm() {
if ! check_param_count_v2 "bucket, key, checksum type, checksum algorithm" 4 $#; then
return 1
fi
if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" CHECKSUM_TYPE="$3" CHECKSUM_ALGORITHM="$4" ./tests/rest_scripts/create_multipart_upload.sh 2>&1); then
log 2 "error creating multipart upload: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/output.txt"))"
return 1
fi
if ! upload_id=$(get_element_text "$TEST_FILE_FOLDER/output.txt" "InitiateMultipartUploadResult" "UploadId"); then
log 2 "error getting upload ID: $upload_id"
return 1
fi
echo "$upload_id"
return 0
}
create_multipart_upload_rest_with_checksum_type_and_algorithm_error() {
if ! check_param_count_v2 "bucket, key, checksum type, checksum algorithm, handle fn, response, code, error" 8 $#; then
return 1
fi
if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" CHECKSUM_TYPE="$3" CHECKSUM_ALGORITHM="$4" ./tests/rest_scripts/create_multipart_upload.sh 2>&1); then
log 2 "error creating multipart upload: $result"
return 1
fi
if ! "$5" "$result" "$TEST_FILE_FOLDER/output.txt" "$6" "$7" "$8"; then
log 2 "error checking result"
return 1
fi
}
create_multipart_upload_s3api() {
record_command "create-multipart-upload" "client:s3api"
if [ $# -ne 2 ]; then
log 2 "create multipart upload function must have bucket, key"
if ! check_param_count_v2 "bucket, key" 2 $#; then
return 1
fi
@@ -37,57 +87,8 @@ create_multipart_upload() {
return 0
}
create_multipart_upload_with_user() {
record_command "create-multipart-upload" "client:s3api"
if [ $# -ne 4 ]; then
log 2 "create multipart upload function must have bucket, key, username, password"
return 1
fi
if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
log 2 "Error creating multipart upload: $multipart_data"
return 1
fi
if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then
log 2 "error parsing upload ID: $upload_id"
return 1
fi
upload_id="${upload_id//\"/}"
echo "$upload_id"
return 0
}
create_multipart_upload_params() {
record_command "create-multipart-upload" "client:s3api"
if [ $# -ne 8 ]; then
log 2 "create multipart upload function with params must have bucket, key, content type, metadata, object lock legal hold status, " \
"object lock mode, object lock retain until date, and tagging"
return 1
fi
local multipart_data
multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload \
--bucket "$1" \
--key "$2" \
--content-type "$3" \
--metadata "$4" \
--object-lock-legal-hold-status "$5" \
--object-lock-mode "$6" \
--object-lock-retain-until-date "$7" \
--tagging "$8" 2>&1) || local create_result=$?
if [[ $create_result -ne 0 ]]; then
log 2 "error creating multipart upload with params: $multipart_data"
return 1
fi
upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq '.UploadId')
upload_id="${upload_id//\"/}"
return 0
}
create_multipart_upload_custom() {
record_command "create-multipart-upload" "client:s3api"
if [ $# -lt 2 ]; then
log 2 "create multipart upload custom function must have at least bucket and key"
create_multipart_upload_s3api_custom() {
if ! check_param_count_gt "at least bucket and key" 2 $#; then
return 1
fi
local multipart_data
@@ -109,24 +110,47 @@ create_multipart_upload_custom() {
return 0
}
create_multipart_upload_rest() {
if [ $# -ne 2 ]; then
log 2 "'create_multipart_upload_rest' requires bucket name, key"
create_multipart_upload_s3api_params() {
record_command "create-multipart-upload" "client:s3api"
if ! check_param_count_v2 "bucket, key, content type, metadata, object lock legal hold status, \
object lock mode, object lock retain until date, and tagging" 8 $#; then
return 1
fi
if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" COMMAND_LOG=$COMMAND_LOG ./tests/rest_scripts/create_multipart_upload.sh); then
log 2 "error creating multipart upload: $result"
local multipart_data
multipart_data=$(send_command aws --no-verify-ssl s3api create-multipart-upload \
--bucket "$1" \
--key "$2" \
--content-type "$3" \
--metadata "$4" \
--object-lock-legal-hold-status "$5" \
--object-lock-mode "$6" \
--object-lock-retain-until-date "$7" \
--tagging "$8" 2>&1) || local create_result=$?
if [[ $create_result -ne 0 ]]; then
log 2 "error creating multipart upload with params: $multipart_data"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "put-object-retention returned code $result: $(cat "$TEST_FILE_FOLDER/output.txt")"
upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq '.UploadId')
upload_id="${upload_id//\"/}"
return 0
}
create_multipart_upload_s3api_with_user() {
record_command "create-multipart-upload" "client:s3api"
if ! check_param_count_v2 "bucket, key, username, password" 4 $#; then
return 1
fi
log 5 "result: $(cat "$TEST_FILE_FOLDER/output.txt")"
if ! upload_id=$(get_element_text "$TEST_FILE_FOLDER/output.txt" "InitiateMultipartUploadResult" "UploadId"); then
log 2 "error getting upload ID: $upload_id"
if ! multipart_data=$(AWS_ACCESS_KEY_ID="$3" AWS_SECRET_ACCESS_KEY="$4" send_command aws --no-verify-ssl s3api create-multipart-upload --bucket "$1" --key "$2" 2>&1); then
log 2 "Error creating multipart upload: $multipart_data"
return 1
fi
if ! upload_id=$(echo "$multipart_data" | grep -v "InsecureRequestWarning" | jq -r '.UploadId' 2>&1); then
log 2 "error parsing upload ID: $upload_id"
return 1
fi
upload_id="${upload_id//\"/}"
echo "$upload_id"
return 0
}

17
tests/commands/delete_object.sh
View File

@@ -45,23 +45,6 @@ delete_object() {
return 0
}
# shellcheck disable=SC2317
delete_object_rest() {
if ! check_param_count "delete_object_rest" "bucket, key" 2 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/delete_object.sh 2>&1); then
log 2 "error deleting object: $result"
return 1
fi
if [ "$result" != "204" ]; then
delete_object_error=$(cat "$TEST_FILE_FOLDER/result.txt")
log 2 "expected '204', was '$result' ($delete_object_error)"
return 1
fi
return 0
}
delete_object_bypass_retention() {
if ! check_param_count "delete_object_bypass_retention" "bucket, key, user, password" 4 $#; then
return 1

15
tests/commands/get_object.sh
View File

@@ -115,3 +115,18 @@ get_object_rest_with_user() {
fi
return 0
}
get_object_rest_with_invalid_streaming_type() {
if ! check_param_count_v2 "bucket, key" 2 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/$2_copy" PAYLOAD="GIBBERISH" ./tests/rest_scripts/get_object.sh 2>&1); then
log 2 "error: $result"
return 1
fi
if ! check_rest_expected_error "$result" "$TEST_FILE_FOLDER/$2_copy" "400" "InvalidArgument" "x-amz-content-sha256 must be"; then
log 2 "error checking response"
return 1
fi
return 0
}

2
tests/commands/get_object_lock_configuration.sh
View File

@@ -14,7 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/drivers/drivers.sh
source ./tests/drivers/params.sh
get_object_lock_configuration() {
record_command "get-object-lock-configuration" "client:s3api"

25
tests/commands/list_buckets.sh
View File

@@ -125,6 +125,29 @@ list_buckets_rest() {
log 2 "list-buckets returned code $result: $(cat "$TEST_FILE_FOLDER/buckets.txt")"
return 1
fi
parse_bucket_list
if ! parse_bucket_list "$TEST_FILE_FOLDER/buckets.txt"; then
log 2 "error parsing bucket list"
return 1
fi
return 0
}
list_buckets_rest_prefix() {
if ! check_param_count_v2 "prefix" 1 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" PREFIX="$1" ./tests/rest_scripts/list_buckets.sh 2>&1); then
log 2 "error getting result: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
log 5 "buckets w/prefix $1: $(cat "$TEST_FILE_FOLDER/result.txt")"
if ! parse_bucket_list "$TEST_FILE_FOLDER/result.txt"; then
log 2 "error parsing bucket list"
return 1
fi
return 0
}

31
tests/commands/put_bucket_acl.sh
View File

@@ -16,6 +16,7 @@
source ./tests/util/util_file.sh
source ./tests/commands/command.sh
source ./tests/drivers/rest.sh
put_bucket_acl_s3api() {
log 6 "put_bucket_acl_s3api"
@@ -128,3 +129,33 @@ put_bucket_acl_rest() {
fi
return 0
}
put_canned_acl_rest() {
if ! check_param_count_v2 "bucket name, canned ACL" 2 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" CANNED_ACL="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_bucket_acl.sh); then
log 2 "error attempting to put bucket acl: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "response code '$result' (message: $(cat "$TEST_FILE_FOLDER/response.txt"))"
return 1
fi
return 0
}
put_bucket_acl_rest_canned_invalid() {
if ! check_param_count_v2 "bucket name, invalid ACL" 2 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" CANNED_ACL="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_bucket_acl.sh); then
log 2 "error attempting to put bucket acl: $result"
return 1
fi
if ! check_rest_expected_error "$result" "$TEST_FILE_FOLDER/response.txt" "400" "InvalidArgument" ""; then
log 2 "error checking REST response (message: $(cat "$TEST_FILE_FOLDER/response.txt"))"
return 1
fi
return 0
}

2
tests/commands/put_bucket_policy.sh
View File

@@ -14,7 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/drivers/drivers.sh
source ./tests/drivers/params.sh
put_bucket_policy() {
log 6 "put_bucket_policy '$1' '$2' '$3'"

16
tests/commands/put_object.sh
View File

@@ -142,6 +142,22 @@ put_object_rest_user_bad_signature() {
return 0
}
put_object_rest_with_unneeded_algorithm_param() {
if ! check_param_count_v2 "local file, bucket name, key, checksum type" 4 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" CHECKSUM_TYPE="$4" \
ALGORITHM_PARAMETER="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_object.sh); then
log 2 "error sending object file: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
return 0
}
put_object_multiple() {
if [ $# -ne 3 ]; then
log 2 "put object command requires command type, source, destination"

69
tests/commands/upload_part.sh
View File

@@ -44,12 +44,11 @@ upload_part_with_user() {
export etag
}
upload_part_and_get_etag_rest() {
if [ $# -ne 5 ]; then
log 2 "'upload_part_rest' requires bucket name, key, part number, upload ID, part"
upload_part_rest() {
if ! check_param_count_v2 "bucket, key, upload ID, part number, part" 5 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="$4" UPLOAD_ID="$3" DATA_FILE="$5" OUTPUT_FILE="$TEST_FILE_FOLDER/etag.txt" ./tests/rest_scripts/upload_part.sh); then
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" PART_NUMBER="$4" DATA_FILE="$5" OUTPUT_FILE="$TEST_FILE_FOLDER/etag.txt" ./tests/rest_scripts/upload_part.sh); then
log 2 "error sending upload-part REST command: $result"
return 1
fi
@@ -60,5 +59,67 @@ upload_part_and_get_etag_rest() {
log 5 "$(cat "$TEST_FILE_FOLDER/etag.txt")"
etag=$(grep -i "etag" "$TEST_FILE_FOLDER/etag.txt" | awk '{print $2}' | tr -d '\r')
log 5 "etag: $etag"
echo "$etag"
return 0
}
upload_part_rest_without_part_number() {
if ! check_param_count_v2 "bucket, key" 2 $#; then
return 1
fi
if ! create_multipart_upload_rest "$1" "$2"; then
log 2 "error creating multpart upload"
return 1
fi
# shellcheck disable=SC2154
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" DATA_FILE="$TEST_FILE_FOLDER/$2" PART_NUMBER="" UPLOAD_ID="$upload_id" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part.sh); then
log 2 "error uploading part $i: $result"
return 1
fi
if ! check_rest_expected_error "$result" "$TEST_FILE_FOLDER/response.txt" "405" "MethodNotAllowed" "method is not allowed"; then
log 2 "error checking error"
return 1
fi
return 0
}
upload_part_rest_without_upload_id() {
if ! check_param_count_v2 "bucket, key" 2 $#; then
return 1
fi
if ! create_multipart_upload_rest "$1" "$2"; then
log 2 "error creating multpart upload"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" DATA_FILE="$TEST_FILE_FOLDER/$2" PART_NUMBER="1" UPLOAD_ID="" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part.sh); then
# shellcheck disable=SC2154
log 2 "error uploading part $i: $result"
return 1
fi
if ! check_rest_expected_error "$result" "$TEST_FILE_FOLDER/response.txt" "405" "MethodNotAllowed" "method is not allowed"; then
log 2 "error checking error"
return 1
fi
return 0
}
upload_part_rest_with_checksum() {
if ! check_param_count_v2 "bucket name, key, upload ID, part number, part, checksum algorithm" 6 $#; then
return 1
fi
# shellcheck disable=SC2154,SC2097,SC2098
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$3" PART_NUMBER="$4" DATA_FILE="$5" CHECKSUM_TYPE="$6" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" OUTPUT_FILE="$TEST_FILE_FOLDER/etag.txt" ./tests/rest_scripts/upload_part.sh); then
log 2 "error sending upload-part REST command: $result"
return 1
fi
if [[ "$result" != "200" ]]; then
log 2 "upload-part command returned error $result: $(cat "$TEST_FILE_FOLDER/etag.txt")"
return 1
fi
log 5 "$(cat "$TEST_FILE_FOLDER/etag.txt")"
etag=$(grep -i "etag" "$TEST_FILE_FOLDER/etag.txt" | awk '{print $2}' | tr -d '\r')
# shellcheck disable=SC2034
checksum=$(grep -i "x-amz-checksum-" "$TEST_FILE_FOLDER/etag.txt" | awk '{print $2}' | tr -d '\r')
log 5 "etag: $etag"
return 0
}

167
tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh Normal file
View File

@@ -0,0 +1,167 @@
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
calculate_multipart_checksum() {
if ! check_param_count_gt "checksum type, part count, data file, checksums" 4 $#; then
return 1
fi
log 5 "checksums: ${*:4}"
if [ "$1" == "COMPOSITE" ]; then
if ! calculate_composite_checksum "$lowercase_checksum_algorithm" ${@:4}; then
log 2 "error calculating checksum"
return 1
fi
checksum="$composite-$2"
return 0
fi
if [ "$1" != "FULL_OBJECT" ]; then
log 2 "unrecognized checksum type: $1"
return 1
fi
if ! checksum=$(DATA_FILE="$3" CHECKSUM_TYPE="$lowercase_checksum_algorithm" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
log 2 "error calculating checksum: $checksum"
return 1
fi
return 0
}
complete_multipart_upload_with_checksum() {
if ! check_param_count_v2 "bucket, key, file, upload ID, part count, checksum type, checksum algorithm" 7 $#; then
return 1
fi
lowercase_checksum_algorithm=$(echo -n "$7" | tr '[:upper:]' '[:lower:]')
if ! upload_parts_rest_with_checksum_before_completion "$1" "$2" "$3" "$4" "$5" "$lowercase_checksum_algorithm"; then
log 2 "error uploading parts"
return 1
fi
log 5 "parts payload: $parts_payload"
log 5 "checksums: ${checksums[*]}"
if ! calculate_multipart_checksum "$6" "$5" "$3" ${checksums[@]}; then
log 2 "error calculating multipart checksum"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$4" PARTS="$parts_payload" CHECKSUM_TYPE="$6" CHECKSUM_ALGORITHM="$7" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then
log 2 "error completing multipart upload"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))"
return 1
fi
log 5 "result: $(cat "$TEST_FILE_FOLDER/result.txt")"
return 0
}
calculate_composite_checksum() {
if ! check_param_count_gt "algorithm, at least two checksums" 3 $#; then
return 1
fi
if ! result=$(truncate -s 0 "$TEST_FILE_FOLDER/all_checksums.bin" 2>&1); then
log 2 "error truncating file: $result"
return 1
fi
log 5 "checksums: ${*:2}"
for checksum in ${@:2}; do
if ! binary_checksum=$(echo -n "$checksum" | base64 -d 2>&1); then
log 2 "error calculating binary checksum: $binary_checksum"
return 1
fi
log 5 "binary checksum: $binary_checksum"
printf "%s" "$binary_checksum" | cat >> "$TEST_FILE_FOLDER/all_checksums.bin"
done
if [ "$1" == "sha256" ]; then
composite=$(openssl dgst -sha256 -binary "$TEST_FILE_FOLDER/all_checksums.bin" | base64)
elif [ "$1" == "sha1" ]; then
composite=$(openssl dgst -sha1 -binary "$TEST_FILE_FOLDER/all_checksums.bin" | base64)
elif [ "$1" == "crc32" ]; then
composite="$(gzip -c -1 "$TEST_FILE_FOLDER/all_checksums.bin" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
elif [ "$1" == "crc32c" ]; then
if ! composite=$(CHECKSUM_TYPE="$1" DATA_FILE="$TEST_FILE_FOLDER/all_checksums.bin" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
log 2 "error calculating crc32c checksum: $composite"
return 1
fi
fi
log 5 "composite: $composite"
}
test_multipart_upload_with_checksum() {
if ! check_param_count_v2 "bucket, filename, checksum type, algorithm" 4 $#; then
return 1
fi
if ! perform_full_multipart_upload_with_checksum_before_completion "$1" "$2" "$3" "$4"; then
log 2 "error performing multipart upload with checksum before completion"
return 1
fi
if ! calculate_multipart_checksum "$3" 2 "$TEST_FILE_FOLDER/$2" ${checksums[@]}; then
log 2 "error calculating multipart checksum"
return 1
fi
if ! complete_multipart_upload_with_checksum "$1" "$2" "$TEST_FILE_FOLDER/$2" "$upload_id" 2 "$3" "$4"; then
log 2 "error completing multipart upload"
return 1
fi
return 0
}
test_complete_multipart_upload_unneeded_algorithm_parameter() {
if ! check_param_count_v2 "bucket, filename, checksum type, algorithm" 4 $#; then
return 1
fi
if ! perform_full_multipart_upload_with_checksum_before_completion "$1" "$2" "$3" "$4"; then
log 2 "error performing multipart upload with checksum before completion"
return 1
fi
if ! complete_multipart_upload_rest_nonexistent_param "$1" "$2" "$upload_id" "$parts_payload"; then
log 2 "error completing multipart upload with nonexistent param"
return 1
fi
return 0
}
test_complete_multipart_upload_incorrect_checksum() {
if ! check_param_count_v2 "bucket, filename, checksum type, algorithm" 4 $#; then
return 1
fi
if ! perform_full_multipart_upload_with_checksum_before_completion "$1" "$2" "$3" "$4"; then
log 2 "error performing multipart upload with checksum before completion"
return 1
fi
if ! calculate_multipart_checksum "$3" 2 "$TEST_FILE_FOLDER/$2" ${checksums[@]}; then
log 2 "error calculating multipart checksum"
return 1
fi
if ! complete_multipart_upload_rest_incorrect_checksum "$1" "$2" "$upload_id" "$parts_payload" "$3" "$4" "$checksum"; then
log 2 "error completing multipart upload with nonexistent param"
return 1
fi
return 0
}
test_complete_multipart_upload_invalid_checksum() {
if ! check_param_count_v2 "bucket, filename, checksum type, algorithm" 4 $#; then
return 1
fi
if ! perform_full_multipart_upload_with_checksum_before_completion "$1" "$2" "$3" "$4"; then
log 2 "error performing multipart upload with checksum before completion"
return 1
fi
if ! complete_multipart_upload_rest_invalid_checksum "$1" "$2" "$upload_id" "$parts_payload" "$3" "$4" "wrong"; then
log 2 "error completing multipart upload with nonexistent param"
return 1
fi
return 0
}

94
tests/drivers/openssl.sh Normal file
View File

@@ -0,0 +1,94 @@
#!/usr/bin/env bats
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/util/util_xml.sh
send_via_openssl() {
if ! check_param_count_v2 "command file" 1 $#; then
return 1
fi
host="${AWS_ENDPOINT_URL#http*://}"
if [[ "$host" =~ s3\..*amazonaws\.com ]]; then
host+=":443"
fi
log 5 "connecting to $host"
if ! result=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then
log 2 "error sending openssl command: $result"
return 1
fi
echo "$result"
}
send_via_openssl_and_check_code() {
if ! check_param_count_v2 "command file, expected code" 2 $#; then
return 1
fi
if ! result=$(send_via_openssl "$1"); then
log 2 "error sending command via openssl"
return 1
fi
response_code="$(echo "$result" | grep "HTTP/" | awk '{print $2}')"
if [ "$response_code" != "$2" ]; then
log 2 "expected '$2', actual '$response_code' (error response: '$result')"
return 1
fi
echo "$result"
}
send_via_openssl_check_code_error_contains() {
if ! check_param_count_v2 "command file, expected code, error, message" 4 $#; then
return 1
fi
if ! result=$(send_via_openssl_and_check_code "$1" "$2"); then
log 2 "error sending and checking code"
return 1
fi
echo -n "$result" > "$TEST_FILE_FOLDER/result.txt"
if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error_data.txt"; then
log 2 "error parsing XML data from result"
return 1
fi
if ! check_xml_error_contains "$TEST_FILE_FOLDER/error_data.txt" "$3" "$4"; then
log 2 "error checking xml error, message"
return 1
fi
return 0
}
send_via_openssl_with_timeout() {
if ! check_param_count_v2 "command file" 1 $#; then
return 1
fi
host="${AWS_ENDPOINT_URL#http*://}"
if [[ "$host" =~ s3\..*amazonaws\.com ]]; then
host+=":443"
fi
log 5 "connecting to $host"
local exit_code=0
result=$(timeout 65 openssl s_client -connect "$host" -ign_eof < "$1" 2>&1) || exit_code=$?
if [ "$exit_code" == 124 ]; then
log 2 "error: openssl command timed out"
return 1
elif [ "$exit_code" != 0 ]; then
log 2 "error sending openssl command: exit code $exit_code, $result"
return 1
fi
if ! [[ "$result" =~ .*$'\nclosed' ]]; then
log 2 "connection not closed properly: $result"
return 1
fi
return 0
}

10
tests/drivers/drivers.sh → tests/drivers/params.sh
View File

@@ -28,7 +28,7 @@ check_param_count() {
check_param_count_v2() {
if [ $# -ne 3 ]; then
log 2 "'check_param_count' requires params list, expected, actual"
log 2 "'check_param_count_v2' requires params list, expected, actual"
return 1
fi
if [ "$2" -ne "$3" ]; then
@@ -51,12 +51,12 @@ assert_param_count() {
}
check_param_count_gt() {
if [ $# -ne 4 ]; then
log 2 "'check_param_count_gt' requires function name, params list, expected minimum, actual"
if [ $# -lt 3 ]; then
log 2 "'check_param_count_gt' requires params list, expected minimum, actual"
return 1
fi
if [ "$3" -gt "$4" ]; then
log_with_stack_ref 2 "function $1 requires $2" 2
if [ "$2" -gt "$3" ]; then
log_with_stack_ref 2 "function '${FUNCNAME[1]}' requires $1" 2
return 1
fi
return 0

32
tests/drivers/rest.sh Normal file
View File

@@ -0,0 +1,32 @@
#!/usr/bin/env bash
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
source ./tests/util/util_xml.sh
check_rest_expected_error() {
if ! check_param_count_v2 "response, response file, expected http code, expected error code, expected error" 5 $#; then
return 1
fi
if [ "$1" != "$3" ]; then
log 2 "expected '$3', was '$1' ($(cat "$2"))"
return 1
fi
if ! check_xml_error_contains "$2" "$4" "$5"; then
log 2 "error checking XML response"
return 1
fi
return 0
}

80
tests/drivers/upload_part/upload_part_rest.sh Normal file
View File

@@ -0,0 +1,80 @@
#!/usr/bin/env bats
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
upload_parts_rest_before_completion() {
if ! check_param_count_v2 "bucket, key, file, upload ID, part count" 5 $#; then
return 1
fi
if ! split_file "$3" "$5"; then
log 2 "error splitting file"
return 1
fi
local parts_payload=""
for ((part=0;part<"$5";part++)); do
part_number=$((part+1))
if ! etag=$(upload_part_rest "$1" "$2" "$4" "$part_number" "$3-$part" 2>&1); then
log 2 "error uploading part $part: $etag"
return 1
fi
parts_payload+="<Part><ETag>$etag</ETag><PartNumber>$part_number</PartNumber></Part>"
done
echo "$parts_payload"
return 0
}
upload_parts_rest_with_checksum_before_completion() {
if ! check_param_count_v2 "bucket, key, file, upload ID, part count, algorithm" 6 $#; then
return 1
fi
if ! split_file "$3" "$5"; then
log 2 "error splitting file"
return 1
fi
parts_payload=""
checksums=()
for ((part=0;part<"$5";part++)); do
part_number=$((part+1))
if ! upload_part_rest_with_checksum "$1" "$2" "$4" "$part_number" "$3-$part" "$6"; then
log 2 "error uploading part $part"
return 1
fi
checksums+=("$checksum")
uppercase_checksum_algorithm=$(echo -n "$6" | tr '[:lower:]' '[:upper:]')
parts_payload+="<Part><ETag>$etag</ETag><Checksum${uppercase_checksum_algorithm}>${checksum}</Checksum${uppercase_checksum_algorithm}><PartNumber>$part_number</PartNumber></Part>"
log 5 "parts payload: $parts_payload"
done
log 5 "${checksums[*]}"
return 0
}
perform_full_multipart_upload_with_checksum_before_completion() {
if ! check_param_count_v2 "bucket, filename, checksum type, algorithm" 4 $#; then
return 1
fi
if ! setup_bucket_and_large_file "$1" "$2"; then
log 2 "error setting up bucket and large file"
return 1
fi
if ! create_multipart_upload_rest_with_checksum_type_and_algorithm "$1" "$2" "$3" "$4"; then
log 2 "error creating multipart upload"
return 1
fi
lowercase_checksum_algorithm=$(echo -n "$4" | tr '[:upper:]' '[:lower:]')
if ! upload_parts_rest_with_checksum_before_completion "$1" "$2" "$TEST_FILE_FOLDER/$2" "$upload_id" 2 "$lowercase_checksum_algorithm"; then
log 2 "error uploading parts"
return 1
fi
}

1
tests/env.sh
View File

@@ -160,6 +160,7 @@ check_universal_vars() {
exit 1
fi
fi
export TEST_FILE_FOLDER
}
delete_command_log() {

10
tests/rest_scripts/calculate_checksum.sh
View File

@@ -45,25 +45,25 @@ calculate_checksum_python() {
}
case "$CHECKSUM_TYPE" in
"crc32c")
"crc32c"|"CRC32C")
if ! checksum=$(calculate_checksum_python "crc32c" "$DATA_FILE" 2>&1); then
log_rest 2 "error getting checksum: $checksum"
exit 1
fi
;;
"crc64nvme")
"crc64nvme"|"CRC64NVME")
if ! checksum=$(calculate_checksum_python "crc64nvme" "$DATA_FILE" 2>&1); then
log 2 "error calculating checksum: $checksum"
exit 1
fi
;;
"sha256")
"sha256"|"SHA256")
checksum="$(sha256sum "$DATA_FILE" | awk '{print $1}' | xxd -r -p | base64)"
;;
"sha1")
"sha1"|"SHA1")
checksum="$(sha1sum "$DATA_FILE" | awk '{print $1}' | xxd -r -p | base64)"
;;
"crc32")
"crc32"|"CRC32")
checksum="$(gzip -c -1 "$DATA_FILE" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
;;
*)

47
tests/rest_scripts/complete_multipart_upload.sh
View File

@@ -26,30 +26,43 @@ key="$OBJECT_KEY"
upload_id="$UPLOAD_ID"
# shellcheck disable=SC2153
parts="$PARTS"
# shellcheck disable=SC2153
checksum_type="$CHECKSUM_TYPE"
# shellcheck disable=SC2153
checksum_algorithm="$CHECKSUM_ALGORITHM"
# shellcheck disable=SC2153
checksum_hash="$CHECKSUM_HASH"
# shellcheck disable=SC2154
algorithm_parameter="${ALGORITHM_PARAMETER:=false}"
payload="<CompleteMultipartUpload xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">$parts</CompleteMultipartUpload>"
payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<CompleteMultipartUpload xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">$parts</CompleteMultipartUpload>"
payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
canonical_request="POST
/$bucket_name/$key
uploadId=$UPLOAD_ID
host:$host
x-amz-content-sha256:$payload_hash
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
$payload_hash"
cr_data=("POST" "/$bucket_name/$key" "uploadId=$upload_id" "host:$host")
log_rest 5 "Algorithm param: $algorithm_parameter"
lowercase_algorithm="$(echo -n "$checksum_algorithm" | tr '[:upper:]' '[:lower:]')"
if [ "$algorithm_parameter" != "false" ]; then
cr_data+=("x-amz-checksum-algorithm:${checksum_algorithm}")
fi
if [ "$checksum_hash" != "" ]; then
cr_data+=("x-amz-checksum-${lowercase_algorithm}:$checksum_hash")
fi
if [ "$checksum_type" != "" ]; then
cr_data+=("x-amz-checksum-type:$checksum_type")
fi
cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
build_canonical_request "${cr_data[@]}"
# shellcheck disable=SC2119
create_canonical_hash_sts_and_signature
curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id"
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
-H "\"x-amz-content-sha256: $payload_hash\""
-H "\"x-amz-date: $current_date_time\""
-H "\"Content-Type: application/xml\""
-d "\"${payload//\"/\\\"}\""
-o "$OUTPUT_FILE")
curl_command+=(curl -iks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploadId=$upload_id"
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
curl_command+=(-H "\"Content-Type: application/xml\"")
curl_command+=("${header_fields[@]}")
curl_command+=(-d "\"${payload//\"/\\\"}\"")
curl_command+=(-o "$OUTPUT_FILE")
# shellcheck disable=SC2154
eval "${curl_command[*]}" 2>&1

33
tests/rest_scripts/create_multipart_upload.sh
View File

@@ -22,30 +22,29 @@ source ./tests/rest_scripts/rest.sh
bucket_name="$BUCKET_NAME"
# shellcheck disable=SC2153
key="$OBJECT_KEY"
# Step 1: generate canonical request hash
# shellcheck disable=SC2153
checksum_type="$CHECKSUM_TYPE"
# shellcheck disable=SC2153
checksum_algorithm="$CHECKSUM_ALGORITHM"
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
canonical_request="POST
/$bucket_name/$key
uploads=
host:$host
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD"
canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')"
cr_data=("POST" "/$bucket_name/$key" "uploads=" "host:$host")
if [ "$checksum_algorithm" != "" ]; then
cr_data+=("x-amz-checksum-algorithm:$checksum_algorithm")
fi
if [ "$checksum_type" != "" ]; then
cr_data+=("x-amz-checksum-type:$checksum_type")
fi
cr_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
build_canonical_request "${cr_data[@]}"
# shellcheck disable=SC2119
create_canonical_hash_sts_and_signature
curl_command+=(curl -ks -w "\"%{http_code}\"" -X POST "$AWS_ENDPOINT_URL/$bucket_name/$key?uploads="
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
-H "\"x-amz-date: $current_date_time\""
-o "$OUTPUT_FILE")
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
curl_command+=("${header_fields[@]}")
curl_command+=(-o "$OUTPUT_FILE")
# shellcheck disable=SC2154
eval "${curl_command[*]}" 2>&1

4
tests/rest_scripts/get_object.sh
View File

@@ -26,6 +26,8 @@ key="$(echo -n "$OBJECT_KEY" | jq -sRr 'split("/") | map(@uri) | join("/")')"
checksum_mode="${CHECKSUM_MODE:=false}"
# shellcheck disable=SC2153
range="$RANGE"
# shellcheck disable=SC2153
payload="${PAYLOAD:=UNSIGNED-PAYLOAD}"
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
@@ -37,7 +39,7 @@ fi
if [ "$checksum_mode" == "true" ]; then
canonical_request_data+=("x-amz-checksum-mode:ENABLED")
fi
canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
canonical_request_data+=("x-amz-content-sha256:$payload" "x-amz-date:$current_date_time")
build_canonical_request "${canonical_request_data[@]}"

6
tests/rest_scripts/put_object.sh
View File

@@ -30,12 +30,12 @@ checksum_type="$CHECKSUM_TYPE"
payload="$PAYLOAD"
# shellcheck disable=SC2153
expires="$EXPIRES"
# use this parameter to check incorrect checksums
# shellcheck disable=SC2153,SC2154
checksum_hash="$CHECKSUM"
# shellcheck disable=SC2153,SC2154
fake_signature="$SIGNATURE"
algorithm_parameter="${ALGORITHM_PARAMETER:=false}"
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
data_file_esc="$(echo -n "$data_file" | sed -e 's/[][`"$^{}]/\\&/g')"
@@ -51,6 +51,9 @@ if [ -n "$expires" ]; then
cr_data+=("expires:$expires")
fi
cr_data+=("host:$host")
if [ "$algorithm_parameter" != "false" ]; then
cr_data+=("x-amz-checksum-algorithm:${checksum_type}")
fi
if [ "$checksum_type" != "" ]; then
if [ "$checksum_hash" == "" ] && ! checksum_hash=$(DATA_FILE="$data_file" CHECKSUM_TYPE="$checksum_type" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
log_rest 2 "error calculating checksum hash"
@@ -68,7 +71,6 @@ if [ "$fake_signature" != "" ]; then
signature="$fake_signature"
fi
curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "\"$AWS_ENDPOINT_URL/$bucket_name/$key\"")
curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
curl_command+=("${header_fields[@]}")

7
tests/rest_scripts/upload_part.go Normal file
View File

@@ -0,0 +1,7 @@
package main
import "fmt"
func main() {
fmt.Println("main")
}

14
tests/rest_scripts/upload_part.sh
View File

@@ -26,6 +26,10 @@ part_number="$PART_NUMBER"
upload_id="$UPLOAD_ID"
# shellcheck disable=SC2153
data=$DATA_FILE
# shellcheck disable=SC2153
checksum_type="$CHECKSUM_TYPE"
# shellcheck disable=SC2153
checksum_hash="$CHECKSUM_HASH"
if [ "$data" != "" ]; then
payload_hash="$(sha256sum "$data" | awk '{print $1}')"
@@ -46,7 +50,15 @@ if [ "$upload_id" != "" ]; then
query_params=$(add_parameter "$query_params" "uploadId=$upload_id")
fi
cr_data+=("$query_params")
cr_data+=("host:$host" "x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
cr_data+=("host:$host")
if [ "$checksum_type" != "" ]; then
if [ "$checksum_hash" == "" ] && ! checksum_hash=$(DATA_FILE="$data" CHECKSUM_TYPE="$checksum_type" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
log_rest 2 "error calculating checksum hash: $checksum_hash"
exit 1
fi
cr_data+=("x-amz-checksum-${checksum_type}:$checksum_hash")
fi
cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
build_canonical_request "${cr_data[@]}"
# shellcheck disable=SC2119

1
tests/setup_static.sh
View File

@@ -14,7 +14,6 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/drivers/drivers.sh
source ./tests/env.sh
source ./tests/util/util_object.sh
source ./tests/commands/create_bucket.sh

2
tests/test_common.sh
View File

@@ -37,7 +37,7 @@ source ./tests/commands/put_bucket_tagging.sh
source ./tests/commands/put_object_tagging.sh
source ./tests/commands/put_object.sh
source ./tests/commands/put_public_access_block.sh
source ./tests/drivers/drivers.sh
source ./tests/drivers/params.sh
# param: command type
# fail on test failure

46
tests/test_rest.sh
View File

@@ -459,3 +459,49 @@ test_file="test_file"
run delete_object_rest "$BUCKET_ONE_NAME" "$file_name/$file_name"
assert_success
}
@test "REST - GetObject w/STREAMING-AWS4-HMAC-SHA256-PAYLOAD type" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1352"
fi
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run get_object_rest_with_invalid_streaming_type "$BUCKET_ONE_NAME" "$test_file"
assert_success
}
@test "REST - PutObject w/x-amz-checksum-algorithm" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1356"
fi
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object_rest_with_unneeded_algorithm_param "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc32c"
assert_success
}
@test "REST - empty message" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1249"
fi
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
echo -en "\r\n" > "$TEST_FILE_FOLDER/empty.txt"
run send_via_openssl_with_timeout "$TEST_FILE_FOLDER/empty.txt"
assert_success
}
@test "REST - deformed message" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1364"
fi
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
echo -en "abcdefg\r\n\r\n" > "$TEST_FILE_FOLDER/deformed.txt"
run send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/deformed.txt" 400 "BadRequest" "An error occurred when parsing the HTTP request."
assert_success
}

15
tests/test_rest_acl.sh
View File

@@ -162,3 +162,18 @@ fi
run put_object_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password"
assert_success
}
@test "REST - invalid canned acl" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1367"
fi
test_file="test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred"
assert_success
run put_bucket_acl_rest_canned_invalid "$BUCKET_ONE_NAME" "privatee"
assert_success
}

15
tests/test_rest_chunked.sh
View File

@@ -216,3 +216,18 @@ source ./tests/util/util_setup.sh
run chunked_upload_trailer_incorrect_checksum "crc64nvme"
assert_success
}
@test "REST chunked upload - smaller chunk size" {
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
test_file="test-file"
run create_test_file "$test_file" 200000
assert_success
run chunked_upload_trailer_different_chunk_size "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "sha256"
assert_success
run download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
assert_success
}

229
tests/test_rest_multipart.sh
View File

@@ -18,6 +18,10 @@ load ./bats-support/load
load ./bats-assert/load
source ./tests/setup.sh
source ./tests/drivers/rest.sh
source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh
source ./tests/drivers/upload_part/upload_part_rest.sh
source ./tests/util/util_file.sh
source ./tests/util/util_list_parts.sh
source ./tests/util/util_setup.sh
@@ -129,7 +133,7 @@ test_file="test_file"
run split_file "$TEST_FILE_FOLDER/$test_file" 4
assert_success
run upload_part_without_upload_id "$BUCKET_ONE_NAME" "$test_file"
run upload_part_rest_without_part_number "$BUCKET_ONE_NAME" "$test_file"
assert_success
}
@@ -143,6 +147,227 @@ test_file="test_file"
run split_file "$TEST_FILE_FOLDER/$test_file" 4
assert_success
run upload_part_without_upload_id "$BUCKET_ONE_NAME" "$test_file"
run upload_part_rest_without_upload_id "$BUCKET_ONE_NAME" "$test_file"
assert_success
}
@test "REST - multipart w/invalid checksum type" {
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run create_multipart_upload_rest_with_checksum_type_and_algorithm_error "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECTS" "" \
check_rest_expected_error "400" "InvalidRequest" "Value for x-amz-checksum-type header is invalid"
assert_success
}
@test "REST - multipart w/invalid checksum algorithm" {
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run create_multipart_upload_rest_with_checksum_type_and_algorithm_error "$BUCKET_ONE_NAME" "$test_file" "" "crc64nvm" \
check_rest_expected_error "400" "InvalidRequest" "Checksum algorithm provided is unsupported."
assert_success
}
@test "REST - multipart checksum w/crc64nvme, composite" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1329"
fi
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run create_multipart_upload_rest_with_checksum_type_and_algorithm_error "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "crc64nvme" \
check_rest_expected_error "400" "InvalidRequest" "The COMPOSITE checksum type cannot be used with the crc64nvme checksum algorithm."
assert_success
}
@test "REST - multipart checksum w/sha1, full object" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1329"
fi
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run create_multipart_upload_rest_with_checksum_type_and_algorithm_error "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "sha1" \
check_rest_expected_error "400" "InvalidRequest" "The FULL_OBJECT checksum type cannot be used with the sha1 checksum algorithm."
assert_success
}
@test "REST - multipart checksum w/sha256, full object" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1329"
fi
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run create_multipart_upload_rest_with_checksum_type_and_algorithm_error "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "sha256" \
check_rest_expected_error "400" "InvalidRequest" "The FULL_OBJECT checksum type cannot be used with the sha256 checksum algorithm."
assert_success
}
@test "REST - multipart - lowercase checksum type and algorithm" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1339"
fi
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run create_multipart_upload_rest_with_checksum_type_and_algorithm "$BUCKET_ONE_NAME" "$test_file" "full_object" "crc64nvme"
assert_success
}
@test "REST - multipart - full object checksum type doesn't require UploadPart checksums" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1342"
fi
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run create_test_file "$test_file" $((5*1024*1024))
assert_success
run create_multipart_upload_rest_with_checksum_type_and_algorithm "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32"
assert_success
upload_id=$output
log 5 "upload ID: $upload_id"
run upload_part_rest "$BUCKET_ONE_NAME" "$test_file" "$upload_id" 1 "$TEST_FILE_FOLDER/$test_file"
assert_success
}
@test "REST - multipart - composite - sha256" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1359"
fi
run test_multipart_upload_with_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "SHA256"
assert_success
}
@test "REST - multipart - composite - sha1" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1359"
fi
run test_multipart_upload_with_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "SHA1"
assert_success
}
@test "REST - multipart - composite - crc32" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1359"
fi
run test_multipart_upload_with_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "CRC32"
assert_success
}
@test "REST - multipart - composite - crc32c" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1359"
fi
run test_multipart_upload_with_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "CRC32C"
assert_success
}
@test "REST - multipart - full object - crc32" {
run test_multipart_upload_with_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32"
assert_success
}
@test "REST - multipart - full object - crc32c" {
run test_multipart_upload_with_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32C"
assert_success
}
@test "REST - multipart - full object - crc64nvme" {
run test_multipart_upload_with_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC64NVME"
assert_success
}
@test "REST - multipart - x-amz-checksum-algorithm is ignored in CompleteMultipartUpload" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1345"
fi
run test_complete_multipart_upload_unneeded_algorithm_parameter "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32C"
assert_success
}
@test "REST - multipart - composite - incorrect sha256" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1359"
fi
run test_complete_multipart_upload_incorrect_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "SHA256"
assert_success
}
@test "REST - multipart - composite - incorrect sha1" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1359"
fi
run test_complete_multipart_upload_incorrect_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "SHA1"
assert_success
}
@test "REST - multipart - composite - incorrect crc32" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1359"
fi
run test_complete_multipart_upload_incorrect_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "CRC32C"
assert_success
}
@test "REST - multipart - composite - incorrect crc32c" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1359"
fi
run test_complete_multipart_upload_incorrect_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "CRC32C"
assert_success
}
@test "REST - multipart - full object - incorrect crc32" {
run test_complete_multipart_upload_incorrect_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32"
assert_success
}
@test "REST - multipart - full object - incorrect crc32c" {
run test_complete_multipart_upload_incorrect_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32C"
assert_success
}
@test "REST - multipart - full object - incorrect crc64nvme" {
run test_complete_multipart_upload_incorrect_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC64NVME"
assert_success
}
@test "REST - multipart - composite - invalid sha1" {
run test_complete_multipart_upload_invalid_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "SHA1"
assert_success
}
@test "REST - multipart - composite - invalid sha256" {
run test_complete_multipart_upload_invalid_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "SHA256"
assert_success
}
@test "REST - multipart - composite - invalid crc32" {
run test_complete_multipart_upload_invalid_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "CRC32"
assert_success
}
@test "REST - multipart - composite - invalid crc32c" {
run test_complete_multipart_upload_invalid_checksum "$BUCKET_ONE_NAME" "$test_file" "COMPOSITE" "CRC32C"
assert_success
}
@test "REST - multipart - full object - invalid crc32" {
run test_complete_multipart_upload_invalid_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32"
assert_success
}
@test "REST - multipart - full object - invalid crc32c" {
run test_complete_multipart_upload_invalid_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32C"
assert_success
}
@test "REST - multipart - full object - invalid crc64nvme" {
run test_complete_multipart_upload_invalid_checksum "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC64NVME"
assert_success
}

2
tests/test_s3api_object.sh
View File

@@ -261,7 +261,7 @@ export RUN_USERS=true
run setup_bucket "$BUCKET_ONE_NAME"
assert_success
run create_multipart_upload "$BUCKET_ONE_NAME" "test_file/"
run create_multipart_upload_s3api "$BUCKET_ONE_NAME" "test_file/"
assert_failure
assert_output -p "Directory object contains data payload"
}

4
tests/test_s3api_policy_multipart.sh
View File

@@ -40,7 +40,7 @@ test_s3api_policy_abort_multipart_upload() {
run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
assert_success
run create_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$username" "$password"
run create_multipart_upload_s3api_with_user "$BUCKET_ONE_NAME" "$test_file" "$username" "$password"
assert_success
# shellcheck disable=SC2154
upload_id="$output"
@@ -82,7 +82,7 @@ test_s3api_policy_list_multipart_uploads() {
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
assert_success
run create_multipart_upload "$BUCKET_ONE_NAME" "$test_file"
run create_multipart_upload_s3api "$BUCKET_ONE_NAME" "$test_file"
assert_success
run list_multipart_uploads_with_user "$BUCKET_ONE_NAME" "$username" "$password"

4
tests/test_user_aws.sh
View File

@@ -158,12 +158,12 @@ export RUN_USERS=true
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run create_multipart_upload_with_user "$BUCKET_ONE_NAME" "dummy" "$username" "$password"
run create_multipart_upload_s3api_with_user "$BUCKET_ONE_NAME" "dummy" "$username" "$password"
assert_failure
run change_bucket_owner "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$BUCKET_ONE_NAME" "$username"
assert_success
run create_multipart_upload_with_user "$BUCKET_ONE_NAME" "dummy" "$username" "$password"
run create_multipart_upload_s3api_with_user "$BUCKET_ONE_NAME" "dummy" "$username" "$password"
assert_success
}

16
tests/util/util_acl.sh
View File

@@ -365,22 +365,6 @@ put_invalid_acl_rest_verify_failure() {
return 0
}
put_canned_acl_rest() {
if [ $# -ne 2 ]; then
log 2 "'put_canned_acl_rest' requires bucket name, canned acl"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" CANNED_ACL="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_bucket_acl.sh); then
log 2 "error attempting to put bucket acl: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "response code '$result' (message: $(cat "$TEST_FILE_FOLDER/response.txt"))"
return 1
fi
return 0
}
# param: bucket name
# return 0 for success, 1 for failure
check_ownership_rule_and_reset_acl() {

6
tests/util/util_bucket.sh
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
source ./tests/drivers/drivers.sh
source ./tests/drivers/params.sh
source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh
source ./tests/drivers/list_objects/list_objects_rest.sh
source ./tests/util/util_acl.sh
@@ -147,7 +147,7 @@ bucket_cleanup() {
# return 0 for success, 1 for error
bucket_cleanup_if_bucket_exists() {
log 6 "bucket_cleanup_if_bucket_exists"
if ! check_param_count_gt "bucket_cleanup_if_bucket_exists" "bucket name, bucket known to exist (optional)" 1 $#; then
if ! check_param_count_gt "bucket name, bucket known to exist (optional)" 1 $#; then
return 1
fi
@@ -170,7 +170,7 @@ bucket_cleanup_if_bucket_exists() {
# params: client, bucket name(s)
# return 0 for success, 1 for failure
setup_buckets() {
if ! check_param_count_gt "setup_buckets" "minimum of 1 bucket name" 1 $#; then
if ! check_param_count_gt "minimum of 1 bucket name" 1 $#; then
return 1
fi
for name in "$@"; do

81
tests/util/util_chunked_upload.sh
View File

@@ -1,8 +1,10 @@
#!/usr/bin/env bash
source ./tests/drivers/rest.sh
source ./tests/drivers/openssl.sh
attempt_seed_signature_without_content_length() {
if [ "$#" -ne 3 ]; then
log 2 "'attempt_seed_signature_without_content_length' requires bucket name, key, data file"
if ! check_param_count_v2 "bucket, key, data file" 3 $#; then
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" \
@@ -53,7 +55,8 @@ attempt_chunked_upload_with_bad_first_signature() {
response_data="${response_data/---/}"
log 5 "response data: $response_data"
log 5 "END"
if ! check_xml_element <(echo "$response_data") "SignatureDoesNotMatch" "Error" "Code"; then
echo -n "$response_data" > "$TEST_FILE_FOLDER/response_data.txt"
if ! check_xml_element "$TEST_FILE_FOLDER/response_data.txt" "SignatureDoesNotMatch" "Error" "Code"; then
log 2 "error checking XML element"
return 1
fi
@@ -106,17 +109,16 @@ attempt_chunked_upload_with_bad_final_signature() {
log 2 "error sending command via openssl"
return 1
fi
response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
log 5 "response code: $response_code"
if [ "$response_code" != "403" ]; then
log 2 "expected code '403', was '$response_code'"
log 5 "response: $result"
echo -n "$result" > "$TEST_FILE_FOLDER/result.txt"
if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error_data.txt"; then
log 2 "error parsing XML data from result"
return 1
fi
response_data="$(echo "$result" | grep "<Error>" | sed 's/---//g')"
log 5 "response data: $response_data"
log 5 "END"
if ! check_xml_element <(echo "$response_data") "SignatureDoesNotMatch" "Error" "Code"; then
log 2 "error checking XML element"
log 5 "xml data: $(cat "$TEST_FILE_FOLDER/error_data.txt")"
response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')"
if ! check_rest_expected_error "$response_code" "$TEST_FILE_FOLDER/error_data.txt" "403" "SignatureDoesNotMatch" "does not match"; then
log 2 "error checking expected REST error"
return 1
fi
return 0
@@ -278,53 +280,24 @@ chunked_upload_trailer_incorrect_checksum() {
return 0
}
send_via_openssl() {
if [ "$#" -ne 1 ]; then
log 2 "'send_via_openssl' requires command file"
chunked_upload_trailer_different_chunk_size() {
if ! check_param_count_v2 "data file, bucket, key, checksum type" 4 $#; then
return 1
fi
host="${AWS_ENDPOINT_URL#http*://}"
if [[ "$host" =~ s3\..*amazonaws\.com ]]; then
host+=":443"
fi
log 5 "connecting to $host"
if ! result=$(openssl s_client -connect "$host" -ign_eof < "$1" 2>&1); then
log 2 "error sending openssl command: $result"
# shellcheck disable=SC2097,SC2098
if ! result=$(COMMAND_LOG="$COMMAND_LOG" \
AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \
DATA_FILE="$1" \
BUCKET_NAME="$2" \
OBJECT_KEY="$3" CHUNK_SIZE=16384 TEST_MODE=false TRAILER="x-amz-checksum-$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then
log 2 "error creating command: $result"
return 1
fi
echo "$result"
}
send_via_openssl_and_check_code() {
if [ "$#" -ne 2 ]; then
log 2 "'send_via_openssl_and_check_code' requires command file, expected code"
return 1
fi
if ! result=$(send_via_openssl "$1"); then
log 2 "error sending command via openssl"
return 1
fi
response_code="$(echo "$result" | grep "HTTP/" | awk '{print $2}')"
if [ "$response_code" != "$2" ]; then
log 2 "expected '$2', actual '$response_code' (error response: '$result')"
return 1
fi
echo "$result"
}
send_via_openssl_check_code_error_contains() {
if [ "$#" -ne 4 ]; then
log 2 "'send_via_openssl_check_code_error_contains' requires command file, expected code, error, message"
return 1
fi
if ! result=$(send_via_openssl_and_check_code "$1" "$2"); then
log 2 "error sending and checking code"
return 1
fi
error_data="$(echo "$result" | grep "<Error>" | sed 's/---//')"
echo -n "$error_data" > "$TEST_FILE_FOLDER/error-data.txt"
if ! check_xml_error_contains "$TEST_FILE_FOLDER/error-data.txt" "$3" "$4"; then
log 2 "error checking xml error, message"
if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then
log 2 "error sending command via openssl or checking response code"
return 1
fi
return 0

7
tests/util/util_delete_object.sh
View File

@@ -27,15 +27,16 @@ delete_object_empty_bucket_check_error() {
log 2 "error getting XML error data: $error"
return 1
fi
if ! check_xml_element <(echo "$error") "MethodNotAllowed" "Code"; then
echo -n "$error" > "$TEST_FILE_FOLDER/error.txt"
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "MethodNotAllowed" "Code"; then
log 2 "Code mismatch"
return 1
fi
if ! check_xml_element <(echo "$error") "POST" "Method"; then
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "POST" "Method"; then
log 2 "Method mismatch"
return 1
fi
if ! check_xml_element <(echo "$error") "SERVICE" "ResourceType"; then
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "SERVICE" "ResourceType"; then
log 2 "ResourceType mismatch"
return 1
fi

4
tests/util/util_file.sh
View File

@@ -244,7 +244,7 @@ create_test_file_count() {
}
download_and_compare_file_with_user() {
if ! check_param_count_gt "download_and_compare_large_file" "original file, bucket, key, destination, username, password, chunk size (optional)" 6 $#; then
if ! check_param_count_gt "original file, bucket, key, destination, username, password, chunk size (optional)" 6 $#; then
return 1
fi
if ! download_file_with_user "$5" "$6" "$2" "$3" "$4" "$7"; then
@@ -260,7 +260,7 @@ download_and_compare_file_with_user() {
download_and_compare_file() {
log 6 "download_and_compare_file"
if ! check_param_count_gt "download_and_compare_file" "original file, bucket, key, destination, chunk size (optional)" 4 $#; then
if ! check_param_count_gt "original file, bucket, key, destination, chunk size (optional)" 4 $#; then
return 1
fi
if ! download_and_compare_file_with_user "$1" "$2" "$3" "$4" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$5"; then

3
tests/util/util_legal_hold.sh
View File

@@ -69,7 +69,8 @@ check_remove_legal_hold_versions() {
return 0
fi
log 5 "legal hold: $legal_hold"
if ! status=$(get_element_text <(echo -n "$legal_hold") "LegalHold" "Status"); then
echo -n "$legal_hold" > "$TEST_FILE_FOLDER/legal_hold.xml"
if ! status=$(get_element_text "$TEST_FILE_FOLDER/legal_hold.xml" "LegalHold" "Status"); then
log 2 "error getting XML legal hold status"
return 1
fi

9
tests/util/util_list_objects.sh
View File

@@ -282,19 +282,20 @@ list_objects_check_params_get_token() {
log 2 "error getting list bucket result: $list_bucket_result"
return 1
fi
if ! check_xml_element <(echo "$list_bucket_result") "$2" "Key"; then
echo -n "$list_bucket_result" > "$TEST_FILE_FOLDER/list_bucket_result.txt"
if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "$2" "Key"; then
log 2 "key mismatch"
return 1
fi
if ! check_xml_element <(echo "$list_bucket_result") "1" "MaxKeys"; then
if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "1" "MaxKeys"; then
log 2 "max keys mismatch"
return 1
fi
if ! check_xml_element <(echo "$list_bucket_result") "1" "KeyCount"; then
if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "1" "KeyCount"; then
log 2 "key count mismatch"
return 1
fi
if ! check_xml_element <(echo "$list_bucket_result") "true" "IsTruncated"; then
if ! check_xml_element "$TEST_FILE_FOLDER/list_bucket_result.txt" "true" "IsTruncated"; then
log 2 "key count mismatch"
return 1
fi

40
tests/util/util_list_parts.sh
View File

@@ -67,38 +67,34 @@ perform_multipart_upload_rest() {
log 2 "'upload_check_parts' requires bucket, key, part list"
return 1
fi
if ! create_upload_and_get_id_rest "$1" "$2"; then
log 2 "error creating upload"
if ! upload_id=$(create_multipart_upload_rest "$1" "$2" 2>&1); then
log 2 "error creating multipart upload"
return 1
fi
# shellcheck disable=SC2154
if ! upload_part_and_get_etag_rest "$1" "$2" "$upload_id" 1 "$3"; then
if ! etag=$(upload_part_rest "$1" "$2" "$upload_id" 1 "$3" 2>&1); then
log 2 "error uploading part 1"
return 1
fi
# shellcheck disable=SC2154
parts_payload="<Part><ETag>$etag</ETag><PartNumber>1</PartNumber></Part>"
if ! upload_part_and_get_etag_rest "$1" "$2" "$upload_id" 2 "$4"; then
log 2 "error uploading part 2"
if ! etag=$(upload_part_rest "$1" "$2" "$upload_id" 2 "$4" 2>&1); then
log 2 "error uploading part 2: $etag"
return 1
fi
parts_payload+="<Part><ETag>$etag</ETag><PartNumber>2</PartNumber></Part>"
if ! upload_part_and_get_etag_rest "$1" "$2" "$upload_id" 3 "$5"; then
log 2 "error uploading part 3"
if ! etag=$(upload_part_rest "$1" "$2" "$upload_id" 3 "$5" 2>&1); then
log 2 "error uploading part 3: $etag"
return 1
fi
parts_payload+="<Part><ETag>$etag</ETag><PartNumber>3</PartNumber></Part>"
if ! upload_part_and_get_etag_rest "$1" "$2" "$upload_id" 4 "$6"; then
log 2 "error uploading part 4"
if ! etag=$(upload_part_rest "$1" "$2" "$upload_id" 4 "$6" 2>&1); then
log 2 "error uploading part 4: $etag"
return 1
fi
parts_payload+="<Part><ETag>$etag</ETag><PartNumber>4</PartNumber></Part>"
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then
log 2 "error completing multipart upload: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
if ! complete_multipart_upload_rest "$1" "$2" "$3" "$4"; then
log 2 "error completing multipart upload"
return 1
fi
return 0
@@ -109,7 +105,7 @@ upload_check_parts() {
log 2 "'upload_check_parts' requires bucket, key, part list"
return 1
fi
if ! create_upload_and_get_id_rest "$1" "$2"; then
if ! upload_id=$(create_multipart_upload_rest "$1" "$2" 2>&1); then
log 2 "error creating upload"
return 1
fi
@@ -144,12 +140,8 @@ upload_check_parts() {
return 1
fi
log 5 "PARTS PAYLOAD: $parts_payload"
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then
log 2 "error completing multipart upload: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then
log 2 "error completing multipart upload"
return 1
fi
return 0
@@ -160,8 +152,8 @@ upload_check_part() {
log 2 "'upload_check_part' requires bucket, key, upload ID, part number, part, etags"
return 1
fi
if ! upload_part_and_get_etag_rest "$1" "$2" "$3" "$4" "$5"; then
log 2 "error uploading part $4"
if ! etag=$(upload_part_rest "$1" "$2" "$3" "$4" "$5" 2>&1); then
log 2 "error uploading part $4: $etag"
return 1
fi
parts_payload+="<Part><ETag>$etag</ETag><PartNumber>$4</PartNumber></Part>"

27
tests/util/util_multipart.sh
View File

@@ -60,7 +60,7 @@ multipart_upload_from_bucket() {
fi
}
if ! create_multipart_upload "$1" "$2-copy"; then
if ! create_multipart_upload_rest "$1" "$2-copy"; then
log 2 "error running first multipart upload"
return 1
fi
@@ -106,7 +106,7 @@ multipart_upload_from_bucket_range() {
fi
}
if ! create_multipart_upload "$1" "$2-copy"; then
if ! create_multipart_upload_rest "$1" "$2-copy"; then
log 2 "error running first multpart upload"
return 1
fi
@@ -131,7 +131,7 @@ multipart_upload_from_bucket_range() {
}
multipart_upload_custom() {
if ! check_param_count_gt "multipart_upload_custom" "bucket, key, file, part count, optional additional parameters" 4 $$; then
if ! check_param_count_gt "bucket, key, file, part count, optional additional parameters" 4 $$; then
return 1
fi
@@ -221,7 +221,7 @@ create_upload_part_copy_rest() {
log 2 "error splitting and putting file"
return 1
fi
if ! create_upload_and_get_id_rest "$1" "$2"; then
if ! create_multipart_upload_rest "$1" "$2"; then
log 2 "error creating upload and getting ID"
return 1
fi
@@ -244,12 +244,8 @@ create_upload_part_copy_rest() {
fi
parts_payload+="<Part><ETag>$etag</ETag><PartNumber>$part_number</PartNumber></Part>"
done
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then
log 2 "error completing multipart upload: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then
log 2 "error completing multipart upload"
return 1
fi
return 0
@@ -262,7 +258,7 @@ create_upload_finish_wrong_etag() {
etag="gibberish"
part_number=1
if ! create_upload_and_get_id_rest "$1" "$2"; then
if ! create_multipart_upload_rest "$1" "$2"; then
log 2 "error creating upload and getting ID"
return 1
fi
@@ -279,19 +275,20 @@ create_upload_finish_wrong_etag() {
log 2 "error retrieving error info: $error"
return 1
fi
if ! check_xml_element <(echo "$error") "InvalidPart" "Code"; then
echo -n "$error" > "$TEST_FILE_FOLDER/error.txt"
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "InvalidPart" "Code"; then
log 2 "code mismatch"
return 1
fi
if ! check_xml_element <(echo "$error") "$upload_id" "UploadId"; then
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "$upload_id" "UploadId"; then
log 2 "upload ID mismatch"
return 1
fi
if ! check_xml_element <(echo "$error") "$part_number" "PartNumber"; then
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "$part_number" "PartNumber"; then
log 2 "part number mismatch"
return 1
fi
if ! check_xml_element <(echo "$error") "$etag" "ETag"; then
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "$etag" "ETag"; then
log 2 "etag mismatch"
return 1
fi

2
tests/util/util_multipart_abort.sh
View File

@@ -43,7 +43,7 @@ create_abort_multipart_upload_rest() {
return 1
fi
log 5 "uploads before upload: $(cat "$TEST_FILE_FOLDER/uploads.txt")"
if ! create_upload_and_get_id_rest "$1" "$2"; then
if ! create_multipart_upload_rest "$1" "$2"; then
log 2 "error creating upload"
return 1
fi

51
tests/util/util_multipart_before_completion.sh
View File

@@ -200,12 +200,12 @@ create_and_list_multipart_uploads() {
return 1
fi
if ! create_multipart_upload "$1" "$2"; then
if ! create_multipart_upload_rest "$1" "$2"; then
log 2 "error creating multpart upload"
return 1
fi
if ! create_multipart_upload "$1" "$3"; then
if ! create_multipart_upload_rest "$1" "$3"; then
log 2 "error creating multpart upload two"
return 1
fi
@@ -243,7 +243,7 @@ multipart_upload_before_completion_with_user() {
return 1
fi
if ! create_multipart_upload_with_user "$1" "$2" "$5" "$6"; then
if ! create_multipart_upload_s3api_with_user "$1" "$2" "$5" "$6"; then
log 2 "error creating multpart upload"
return 1
fi
@@ -276,7 +276,7 @@ multipart_upload_before_completion_with_params() {
return 1
fi
if ! create_multipart_upload_params "$1" "$2" "$5" "$6" "$7" "$8" "$9" "${10}"; then
if ! create_multipart_upload_s3api_params "$1" "$2" "$5" "$6" "$7" "$8" "$9" "${10}"; then
log 2 "error creating multpart upload"
return 1
fi
@@ -331,31 +331,8 @@ multipart_upload_before_completion_custom() {
export parts
}
create_upload_and_get_id_rest() {
if [ $# -ne 2 ]; then
log 2 "'create_upload_and_get_id_rest' requires bucket, key"
return 1
fi
if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY=$2 OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" ./tests/rest_scripts/create_multipart_upload.sh); then
log 2 "error creating multipart upload: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "error: response code: $result, output: $(cat "$TEST_FILE_FOLDER/output.txt")"
return 1
fi
log 5 "multipart upload create info: $(cat "$TEST_FILE_FOLDER/output.txt")"
if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/output.txt" 2>&1); then
log 2 "error getting upload ID: $upload_id"
return 1
fi
log 5 "upload ID: $upload_id"
return 0
}
multipart_upload_range_too_large() {
if [ $# -ne 3 ]; then
log 2 "'multipart_upload_range_too_large' requires bucket name, key, file location"
if ! check_param_count_v2 "bucket, key, file location" 3 $#; then
return 1
fi
if multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-1000000000"; then
@@ -462,16 +439,10 @@ upload_part_check_etag_header() {
log 2 "'upload_part_check_etag_header' requires bucket name, key, upload ID"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="1" UPLOAD_ID="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part.sh); then
# shellcheck disable=SC2154
log 2 "error uploading part $i: $result"
if ! etag=$(upload_part_rest "$1" "$2" "$3" 1 2>&1); then
log 2 "error getting etag: $etag"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected '200', was '$result'"
return 1
fi
etag="$(grep -i "ETag: " "$TEST_FILE_FOLDER/response.txt" | awk '{print $2}' | tr -d '\r')"
if ! [[ "$etag" =~ ^\"[0-9a-f]+\" ]]; then
log 2 "etag pattern mismatch, etag ($etag) should be hex string surrounded by quotes"
return 1
@@ -484,8 +455,8 @@ upload_part_copy_check_etag_header() {
log 2 "'upload_part_copy_check_etag_header' requires bucket, destination file, part location"
return 1
fi
if ! create_upload_and_get_id_rest "$1" "$2"; then
log 2 "error creating upload and getting ID"
if ! upload_id=$(create_multipart_upload_rest "$1" "$2" 2>&1); then
log 2 "error creating upload and getting ID: $upload_id"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="1" UPLOAD_ID="$upload_id" PART_LOCATION="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part_copy.sh); then
@@ -535,7 +506,7 @@ upload_part_without_upload_id() {
log 2 "error creating multpart upload"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" DATA_FILE="$TEST_FILE_FOLDER/$2" PART_NUMBER="1" UPLOAD_ID="" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part.sh); then
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" DATA_FILE="$TEST_FILE_FOLDER/$2" PART_NUMBER="1" UPLOAD_ID="" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part.sh 2>&1); then
# shellcheck disable=SC2154
log 2 "error uploading part $i: $result"
return 1
@@ -545,4 +516,4 @@ upload_part_without_upload_id() {
return 1
fi
return 0
}
}

2
tests/util/util_object.sh
View File

@@ -404,7 +404,7 @@ put_object_rest_check_expires_header() {
}
download_file_with_user() {
if ! check_param_count_gt "download_large_file" "username, password, bucket, key, destination, chunk size (optional)" 5 $#; then
if ! check_param_count_gt "username, password, bucket, key, destination, chunk size (optional)" 5 $#; then
return 1
fi
if ! file_size=$(get_object_size_with_user "$1" "$2" "$3" "$4" 2>&1); then

7
tests/util/util_rest.sh
View File

@@ -1,9 +1,12 @@
#!/usr/bin/env bash
parse_bucket_list() {
if ! check_param_count_v2 "data file" 1 $#; then
return 1
fi
# shellcheck disable=SC2154
log 5 "bucket list: $(cat "$TEST_FILE_FOLDER/buckets.txt")"
bucket_list=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$TEST_FILE_FOLDER/buckets.txt")
log 5 "bucket list: $(cat "$1")"
bucket_list=$(xmllint --xpath '//*[local-name()="Bucket"]/*[local-name()="Name"]/text()' "$1")
bucket_array=()
while read -r bucket; do
bucket_array+=("$bucket")

2
tests/util/util_retention.sh
View File

@@ -14,7 +14,7 @@
# specific language governing permissions and limitations
# under the License.
source ./tests/drivers/drivers.sh
source ./tests/drivers/params.sh
# params: bucket name
# return 0 for success, 1 for error

4
tests/util/util_setup.sh
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
source ./tests/drivers/drivers.sh
source ./tests/drivers/params.sh
setup_bucket_and_file() {
if ! check_param_count "setup_bucket_and_file" "bucket, file name" 2 $#; then
@@ -14,7 +14,7 @@ setup_bucket_and_file() {
}
setup_bucket_and_files() {
if ! check_param_count_gt "setup_bucket_and_files" "bucket, file name" 2 $#; then
if ! check_param_count_gt "bucket, file name" 2 $#; then
return 1
fi
if ! setup_bucket "$1"; then

4
tests/util/util_versioning.sh
View File

@@ -2,7 +2,7 @@
source ./tests/commands/get_bucket_versioning.sh
source ./tests/commands/list_object_versions.sh
source ./tests/drivers/drivers.sh
source ./tests/drivers/params.sh
check_if_versioning_enabled() {
if ! check_param_count "check_if_versioning_enabled" "bucket" 1 $#; then
@@ -203,7 +203,7 @@ parse_versions_rest() {
}
get_and_check_versions_rest() {
if ! check_param_count_gt "get_and_check_versions_rest" "bucket, key, count, expected islatest, expected id equal to null" 5 $#; then
if ! check_param_count_gt "bucket, key, count, expected islatest, expected id equal to null" 5 $#; then
return 1
fi
if ! list_object_versions_rest "$1"; then

85
tests/util/util_xml.sh
View File

@@ -1,16 +1,45 @@
#!/usr/bin/env bash
get_element_text() {
if [ $# -lt 2 ]; then
log 2 "'get_element_text' requires data source, XML tree"
build_xpath_string() {
if ! check_param_count_gt "XML tree" 1 $#; then
return 1
fi
local xpath='//'
for tree_val in "${@:2}"; do
xpath='//'
for tree_val in "$@"; do
xpath+='*[local-name()="'$tree_val'"]/'
done
xpath+='text()'
if ! xml_val=$(xmllint --xpath "$xpath" "$1" 2>&1); then
}
check_for_empty_element() {
if ! check_param_count_gt "data file, XML tree" 2 $#; then
return 1
fi
# shellcheck disable=SC2068
if ! build_xpath_string ${@:2}; then
log 2 "error building XPath search string"
return 1
fi
if grep '<[^/][^ >]*>' "$1" | xmllint --xpath "'${xpath}[not(normalize-space())]'" -; then
return 0
fi
return 1
}
get_element_text() {
if [ $# -lt 2 ]; then
log 2 "'get_element_text' requires data file, XML tree"
return 1
fi
if ! build_xpath_string "${@:2}"; then
log 2 "error building XPath search string"
return 1
fi
log 5 "data: $(cat "$1")"
if ! xml_val=$(grep '<[^/][^ >]*>' "$1" | xmllint --xpath "$xpath" - 2>&1); then
log 2 "error getting XML value matching $xpath: $xml_val (file data: $(cat "$1"))"
return 1
fi
@@ -38,9 +67,17 @@ check_xml_element_contains() {
log 2 "'check_xml_element_contains' requires data source, expected value, XML tree"
return 1
fi
if ! xml_val=$(get_element_text "$1" "${@:3}"); then
log 2 "error getting element text"
return 1
if [ "$2" == "" ]; then
if ! check_for_empty_element "$1" "${@:3}"; then
log 2 "Message value not empty"
return 1
fi
return 0
else
if ! xml_val=$(get_element_text "$1" "${@:3}"); then
log 2 "error getting element text"
return 1
fi
fi
if [[ "$xml_val" != *"$2"* ]]; then
log 2 "XML data mismatch, expected '$2', actual '$xml_val'"
@@ -64,3 +101,33 @@ check_xml_error_contains() {
fi
return 0
}
get_xml_data() {
if ! check_param_count_v2 "data file, output file" 2 $#; then
return 1
fi
log 5 "data: $(cat "$1")"
# Find first line with "<?xml" and everything from there onward
xml_start=$(grep -n "<?xml" "$1" | head -n 1 | cut -d: -f1)
if [ -z "$xml_start" ]; then
log 2 "No XML declaration found."
return 1
fi
log 5 "xml start: $xml_start"
# Grab everything from the XML start line to the end of the file
tail -n +"$xml_start" "$1" > "$2"
log 5 "xml data after start: $(cat "$2")"
# Try to extract valid XML using xmllint recover mode
# This will truncate anything after the root closing tag
truncated=$(xmllint --recover --noent --nocdata "$2" 2>/dev/null |
awk 'BEGIN{xml=0}
/<\?xml/{xml=1}
{if (xml) print}
/<\/[^>]+>/{lastline=NR}
END{exit}')
echo -n "$truncated" > "$2"
}