mirror of
https://github.com/versity/versitygw.git
synced 2026-09-21 07:24:29 +00:00
test: website redirect test, multipart upload util cleanup, install updates
This commit is contained in:
Luke McCrone
@@ -36,6 +36,8 @@ ENV PATH="$VENV/bin:${PATH}"
|
||||
WORKDIR /home/tester
|
||||
RUN make
|
||||
|
||||
RUN chown -R tester:tester /home/tester/.cache/go-build /home/tester/go
|
||||
|
||||
USER tester
|
||||
|
||||
RUN . $SECRETS_FILE && \
|
||||
@@ -55,5 +57,14 @@ RUN mkdir config log && cp $CONFIG_FILE config/.env
|
||||
ENV WORKSPACE=.
|
||||
ENV VERSITYGW_TEST_ENV=/home/tester/config/.env
|
||||
|
||||
#RUN set -x; \
|
||||
# id; \
|
||||
# whoami; \
|
||||
# echo "GOCACHE=$(go env GOCACHE)"; \
|
||||
# echo "GOMODCACHE=$(go env GOMODCACHE)"; \
|
||||
# echo "GOPATH=$(go env GOPATH)" || true
|
||||
#
|
||||
#RUN cache="$(go env GOCACHE)" && ls -ld "$cache" && find "$cache" -maxdepth 2 -ls 2>/dev/null | head -100 || true
|
||||
|
||||
ENTRYPOINT ["tests/run.sh"]
|
||||
CMD ["s3api,s3,s3cmd,mc,rest"]
|
||||
|
||||
@@ -90,37 +90,3 @@ put_bucket_policy_rest_200_or_204() {
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
create_website_with_random_string_and_add_permissions() {
|
||||
if ! check_param_count_v2 "bucket name" 1 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket_name="$1"
|
||||
local response random_string policy_file
|
||||
|
||||
if ! response=$(create_website_with_random_string "$bucket_name" 2>&1); then
|
||||
log 2 "error creating website: $response"
|
||||
return 1
|
||||
fi
|
||||
read -r _ random_string <<< "$response"
|
||||
|
||||
if [ "$DIRECT" == "true" ]; then
|
||||
if ! put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false"; then
|
||||
log 2 "error putting public access block"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! response=$(setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" 2>&1); then
|
||||
log 2 "error setting up policy: $response"
|
||||
return 1
|
||||
fi
|
||||
policy_file="$response"
|
||||
|
||||
if ! put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"; then
|
||||
log 2 "error putting bucket policy"
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$random_string"
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -15,34 +15,37 @@
|
||||
# under the License.
|
||||
|
||||
upload_part_copy() {
|
||||
if [ $# -ne 5 ]; then
|
||||
log 2 "upload multipart part copy function must have bucket, key, upload ID, file name, part number"
|
||||
if ! check_param_count_v2 "bucket, key, upload ID, file name, part number" 5 $#; then
|
||||
return 1
|
||||
fi
|
||||
local etag_json
|
||||
log 5 "parameters: '$1' '$2' '$3' '$4' '$5'"
|
||||
etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$?
|
||||
if [[ $uploaded -ne 0 ]]; then
|
||||
log 2 "Error uploading part $5: $etag_json"
|
||||
local bucket="$1" key="$2" upload_id="$3" file_name="$4" part_number="$5"
|
||||
local response response_without_warning
|
||||
|
||||
if ! response=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" 2>&1); then
|
||||
log 2 "UploadPartCopy command error: $response"
|
||||
return 1
|
||||
fi
|
||||
etag=$(echo "$etag_json" | jq '.CopyPartResult.ETag')
|
||||
export etag
|
||||
log 5 "UploadPartCopy response: $response"
|
||||
response_without_warning=$(grep -v "InsecureRequestWarning" - <<< "$response")
|
||||
etag=$(jq -r '.CopyPartResult.ETag' - <<< "$response_without_warning" 2>&1)
|
||||
printf '%s\n' "$etag"
|
||||
return 0
|
||||
}
|
||||
|
||||
upload_part_copy_with_range() {
|
||||
if [ $# -ne 6 ]; then
|
||||
log 2 "upload multipart part copy function must have bucket, key, upload ID, file name, part number, range"
|
||||
if ! check_param_count_v2 "bucket, key, upload ID, file name, part number, range" 6 $#; then
|
||||
return 1
|
||||
fi
|
||||
local etag_json
|
||||
log 5 "bucket: $1, key: $2, upload ID: $3, file name: $4, range: $5, copy source range: $6"
|
||||
etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" --copy-source-range "$6" 2>&1) || local uploaded=$?
|
||||
if [[ $uploaded -ne 0 ]]; then
|
||||
log 2 "Error uploading part $5: $etag_json"
|
||||
export upload_part_copy_error=$etag_json
|
||||
local bucket="$1" key="$2" upload_id="$3" file_name="$4" part_number="$5" range="$6"
|
||||
local response etag
|
||||
|
||||
if ! response=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$bucket" --key "$key" --upload-id "$upload_id" \
|
||||
--part-number "$part_number" --copy-source "${bucket}/${file_name}-$((part_number-1))" --copy-source-range "$range" 2>&1); then
|
||||
printf "s3api UploadPartCopy command error: %s\n" "$response"
|
||||
return 1
|
||||
fi
|
||||
etag=$(echo "$etag_json" | grep -v "InsecureRequestWarning" | jq '.CopyPartResult.ETag')
|
||||
export etag
|
||||
log 5 "UploadPartCopy with range response: $response"
|
||||
etag=$(echo "$response" | grep -v "InsecureRequestWarning" | jq -r '.CopyPartResult.ETag')
|
||||
printf '%s\n' "$etag"
|
||||
return 0
|
||||
}
|
||||
@@ -13,6 +13,15 @@ download_apt_packages() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! env DEBIAN_FRONTEND=noninteractive sudo rm -rf /var/lib/apt/lists/*; then
|
||||
echo "error removing old apt lists" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! env DEBIAN_FRONTEND=noninteractive sudo apt-get clean; then
|
||||
echo "error with apt clean" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! env DEBIAN_FRONTEND=noninteractive sudo apt-get update; then
|
||||
echo "error with apt update" >&2
|
||||
return 1
|
||||
|
||||
@@ -18,22 +18,25 @@ calculate_multipart_checksum() {
|
||||
if ! check_param_count_gt "checksum type, part count, data file, lowercase algorithm, checksums" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
log 5 "checksums: ${*:5}"
|
||||
if [ "$1" == "COMPOSITE" ]; then
|
||||
if ! response=$(calculate_composite_checksum "$4" "${@:5}" 2>&1); then
|
||||
local checksum_type="$1" part_count="$2" data_file="$3" lowercase_algorithm="$4" checksums=("${@:5}")
|
||||
local response
|
||||
|
||||
log 5 "checksums: ${checksums[*]}"
|
||||
if [ "$checksum_type" == "COMPOSITE" ]; then
|
||||
if ! response=$(calculate_composite_checksum "$lowercase_algorithm" "${checksums[@]}" 2>&1); then
|
||||
log 2 "error calculating checksum: $response"
|
||||
return 1
|
||||
fi
|
||||
checksum="$response-$2"
|
||||
checksum="$response-${part_count}"
|
||||
echo "$checksum"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$1" != "FULL_OBJECT" ]; then
|
||||
log 2 "unrecognized checksum type: $1"
|
||||
if [ "$checksum_type" != "FULL_OBJECT" ]; then
|
||||
log 2 "unrecognized checksum type: '$checksum_type'"
|
||||
return 1
|
||||
fi
|
||||
if ! response=$(DATA_FILE="$3" CHECKSUM_TYPE="$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
|
||||
if ! response=$(DATA_FILE="$data_file" CHECKSUM_TYPE="$lowercase_algorithm" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
|
||||
log 2 "error calculating checksum: $response"
|
||||
return 1
|
||||
fi
|
||||
@@ -42,17 +45,21 @@ calculate_multipart_checksum() {
|
||||
}
|
||||
|
||||
complete_multipart_upload_with_checksum() {
|
||||
if ! check_param_count_v2 "bucket, key, file, upload ID, part count, parts payload, checksum type, checksum algorithm" 8 $#; then
|
||||
if ! check_param_count_v2 "bucket, key, file, upload ID, part count, parts payload, checksum type, checksum algorithm, checksum" 9 $#; then
|
||||
return 1
|
||||
fi
|
||||
log 5 "checksum algorithm: $8"
|
||||
local bucket="$1" key="$2" file="$3" upload_id="$4" part_count="$5" parts_payload="$6" checksum_type="$7" checksum_algorithm="$8" checksum="$9"
|
||||
local response response_file result
|
||||
|
||||
log 5 "checksum algorithm: '$checksum_algorithm'"
|
||||
if ! response=$(get_file_name 2>&1); then
|
||||
log 2 "error getting file name: $response"
|
||||
return 1
|
||||
fi
|
||||
response_file="$response"
|
||||
|
||||
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$4" PARTS="$6" CHECKSUM_TYPE="$7" CHECKSUM_ALGORITHM="$8" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/$response_file" ./tests/rest_scripts/complete_multipart_upload.sh); then
|
||||
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" UPLOAD_ID="$upload_id" PARTS="$parts_payload" CHECKSUM_TYPE="$checksum_type" \
|
||||
CHECKSUM_ALGORITHM="$checksum_algorithm" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/$response_file" ./tests/rest_scripts/complete_multipart_upload.sh); then
|
||||
log 2 "error completing multipart upload"
|
||||
return 1
|
||||
fi
|
||||
@@ -87,7 +94,7 @@ calculate_composite_checksum() {
|
||||
elif [ "$1" == "crc32" ]; then
|
||||
composite="$(gzip -c -1 "$TEST_FILE_FOLDER/all_checksums.bin" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
|
||||
elif [ "$1" == "crc32c" ]; then
|
||||
if ! composite=$(CHECKSUM_TYPE="$1" DATA_FILE="$TEST_FILE_FOLDER/all_checksums.bin" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
|
||||
if ! composite=$(CHECKSUM_TYPE="$1" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" DATA_FILE="$TEST_FILE_FOLDER/all_checksums.bin" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then
|
||||
log 2 "error calculating crc32c checksum: $composite"
|
||||
return 1
|
||||
fi
|
||||
@@ -102,6 +109,9 @@ test_multipart_upload_with_checksum() {
|
||||
if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then
|
||||
return 1
|
||||
fi
|
||||
local checksum_type="$1" checksum_algorithm="$2"
|
||||
local file_and_bucket bucket_name mp_file_name response checksum
|
||||
|
||||
if ! file_and_bucket=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
|
||||
log 2 "error setting up file and bucket"
|
||||
return 1
|
||||
@@ -109,7 +119,7 @@ test_multipart_upload_with_checksum() {
|
||||
read -r bucket_name mp_file_name <<< "$file_and_bucket"
|
||||
log 5 "file name: $mp_file_name, file info: $(ls -l "$TEST_FILE_FOLDER/$mp_file_name")"
|
||||
|
||||
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then
|
||||
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$checksum_type" "$checksum_algorithm" 2>&1); then
|
||||
log 2 "error performing multipart upload with checksum before completion: $response"
|
||||
return 1
|
||||
fi
|
||||
@@ -120,11 +130,13 @@ test_multipart_upload_with_checksum() {
|
||||
local payload_parts="${response_lines[3]}"
|
||||
read -r -a checksums <<< "$checksum_string"
|
||||
|
||||
if ! calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}"; then
|
||||
log 2 "error calculating multipart checksum"
|
||||
if ! response=$(calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}" 2>&1); then
|
||||
log 2 "error calculating multipart checksum: $response"
|
||||
return 1
|
||||
fi
|
||||
if ! complete_multipart_upload_with_checksum "$bucket_name" "$mp_file_name" "$TEST_FILE_FOLDER/$mp_file_name" "$upload_id" 2 "$payload_parts" "$1" "$2"; then
|
||||
checksum="$response"
|
||||
|
||||
if ! complete_multipart_upload_with_checksum "$bucket_name" "$mp_file_name" "$TEST_FILE_FOLDER/$mp_file_name" "$upload_id" 2 "$payload_parts" "$1" "$2" "$checksum"; then
|
||||
log 2 "error completing multipart upload with checksum"
|
||||
return 1
|
||||
fi
|
||||
@@ -135,12 +147,16 @@ test_complete_multipart_upload_unneeded_algorithm_parameter() {
|
||||
if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then
|
||||
return 1
|
||||
fi
|
||||
local checksum_type="$1" algorithm="$2"
|
||||
local file_and_bucket bucket_name mp_file_name
|
||||
lcoal -a response_lines
|
||||
|
||||
if ! file_and_bucket=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
|
||||
log 2 "error setting up file and bucket"
|
||||
return 1
|
||||
fi
|
||||
read -r bucket_name mp_file_name <<< "$file_and_bucket"
|
||||
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then
|
||||
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$checksum_type" "$algorithm" 2>&1); then
|
||||
log 2 "error performing multipart upload with checksum before completion"
|
||||
return 1
|
||||
fi
|
||||
@@ -159,29 +175,33 @@ test_complete_multipart_upload_incorrect_checksum() {
|
||||
if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then
|
||||
return 1
|
||||
fi
|
||||
local checksum_type="$1" algorithm="$2"
|
||||
local response bucket_name mp_file_name checksum
|
||||
local -a response_lines
|
||||
|
||||
if ! response=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
|
||||
log 2 "error setting up file and bucket"
|
||||
return 1
|
||||
fi
|
||||
read -r bucket_name mp_file_name <<< "$response"
|
||||
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then
|
||||
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$checksum_type" "$algorithm" 2>&1); then
|
||||
log 2 "error performing multipart upload with checksum before completion"
|
||||
return 1
|
||||
fi
|
||||
mapfile -t response_lines<<< "$response"
|
||||
mapfile -t response_lines <<< "$response"
|
||||
local lowercase_checksum_algorithm="${response_lines[0]}"
|
||||
local upload_id="${response_lines[1]}"
|
||||
local checksum_string="${response_lines[2]}"
|
||||
local payload_parts="${response_lines[3]}"
|
||||
read -r -a checksums <<< "$checksum_string"
|
||||
|
||||
if ! response=$(calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}" 2>&1); then
|
||||
if ! response=$(calculate_multipart_checksum "$checksum_type" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}" 2>&1); then
|
||||
log 2 "error calculating multipart checksum"
|
||||
return 1
|
||||
fi
|
||||
checksum="$response"
|
||||
|
||||
if ! complete_multipart_upload_rest_incorrect_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$payload_parts" "$1" "$2" "$checksum"; then
|
||||
if ! complete_multipart_upload_rest_incorrect_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$payload_parts" "$checksum_type" "$algorithm" "$checksum"; then
|
||||
log 2 "error completing multipart upload with incorrect checksum"
|
||||
return 1
|
||||
fi
|
||||
@@ -192,12 +212,16 @@ test_complete_multipart_upload_invalid_checksum() {
|
||||
if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then
|
||||
return 1
|
||||
fi
|
||||
local checksum_type="$1" algorithm="$2"
|
||||
local file_and_bucket bucket_name mp_file_name response
|
||||
local -a response_lines
|
||||
|
||||
if ! file_and_bucket=$(setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" 2>&1); then
|
||||
log 2 "error setting up file and bucket"
|
||||
return 1
|
||||
fi
|
||||
read -r bucket_name mp_file_name <<< "$file_and_bucket"
|
||||
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then
|
||||
if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$checksum_type" "$algorithm" 2>&1); then
|
||||
log 2 "error performing multipart upload with checksum before completion: $response"
|
||||
return 1
|
||||
fi
|
||||
@@ -205,7 +229,7 @@ test_complete_multipart_upload_invalid_checksum() {
|
||||
local upload_id="${response_lines[1]}"
|
||||
local parts_payload="${response_lines[3]}"
|
||||
|
||||
if ! complete_multipart_upload_rest_invalid_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload" "$1" "$2" "wrong"; then
|
||||
if ! complete_multipart_upload_rest_invalid_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload" "$checksum_type" "$algorithm" "wrong"; then
|
||||
log 2 "error completing multipart upload with invalid checksum"
|
||||
return 1
|
||||
fi
|
||||
@@ -216,13 +240,15 @@ complete_multipart_upload_invalid_object_size_string() {
|
||||
if ! check_param_count_v2 "bucket, key, file" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" key="$2" file="$3"
|
||||
local response upload_id parts_payload
|
||||
if ! response=$(multipart_upload_rest_before_completion "$1" "$2" "$3" 2 2>&1); then
|
||||
|
||||
if ! response=$(multipart_upload_rest_before_completion "$bucket" "$key" "$file" 2 2>&1); then
|
||||
log 2 "error performing multipart upload before completion: $response"
|
||||
return 1
|
||||
fi
|
||||
read -r upload_id parts_payload <<< "$response"
|
||||
if ! complete_multipart_upload_rest_expect_error "$1" "$2" "$upload_id" "$parts_payload" "MULTIPART_OBJECT_SIZE=size" "400" "InvalidRequest" "Value for x-amz-mp-object-size header is invalid"; then
|
||||
if ! complete_multipart_upload_rest_expect_error "$bucket" "$key" "$upload_id" "$parts_payload" "MULTIPART_OBJECT_SIZE=size" "400" "InvalidRequest" "Value for x-amz-mp-object-size header is invalid"; then
|
||||
log 2 "error completing multipart upload"
|
||||
return 1
|
||||
fi
|
||||
@@ -233,7 +259,9 @@ perform_multipart_upload_rest() {
|
||||
if ! check_param_count_v2 "bucket, key, four parts" 6 $#; then
|
||||
return 1
|
||||
fi
|
||||
if ! perform_multipart_upload_rest_variable_parts "$@"; then
|
||||
local bucket="$1" key="$2" four_parts=("${@:3}")
|
||||
|
||||
if ! perform_multipart_upload_rest_variable_parts "$bucket" "$key" "${four_parts[@]}"; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
@@ -243,20 +271,22 @@ upload_check_parts() {
|
||||
if ! check_param_count_v2 "bucket, key, list of 4 parts" 6 $#; then
|
||||
return 1
|
||||
fi
|
||||
if ! upload_id=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then
|
||||
local bucket="$1" key="$2" four_parts=("${@:3}")
|
||||
local upload_id parts_payload
|
||||
|
||||
if ! upload_id=$(create_multipart_upload_rest "$bucket" "$key" "" "parse_upload_id" 2>&1); then
|
||||
log 2 "error creating upload: $upload_id"
|
||||
return 1
|
||||
fi
|
||||
if ! check_part_list_rest "$1" "$2" "$upload_id" 0; then
|
||||
if ! check_part_list_rest "$bucket" "$key" "$upload_id" 0; then
|
||||
log 2 "error checking part list before part upload"
|
||||
return 1
|
||||
fi
|
||||
if ! parts_payload=$(upload_each_part_and_check "$1" "$2" "$upload_id" "${@:3}" 2>&1); then
|
||||
if ! parts_payload=$(upload_each_part_and_check "$bucket" "$key" "$upload_id" "${four_parts[@]}" 2>&1); then
|
||||
log 2 "error uploading and checking parts: $parts_payload"
|
||||
return 1
|
||||
fi
|
||||
log 5 "PARTS PAYLOAD: $parts_payload"
|
||||
if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then
|
||||
if ! complete_multipart_upload_rest "$bucket" "$key" "$upload_id" "$parts_payload"; then
|
||||
log 2 "error completing multipart upload"
|
||||
return 1
|
||||
fi
|
||||
@@ -267,17 +297,18 @@ perform_multipart_upload_rest_variable_parts() {
|
||||
if ! check_param_count_gt "bucket, key, at least two part locations" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
local response
|
||||
local bucket="$1" key="$2" parts=("${@:3}")
|
||||
local response upload_id part etag
|
||||
|
||||
if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then
|
||||
if ! response=$(create_multipart_upload_rest "$bucket" "$key" "" "parse_upload_id" 2>&1); then
|
||||
log 2 "error creating multipart upload: $response"
|
||||
return 1
|
||||
fi
|
||||
upload_id="$response"
|
||||
|
||||
local parts_payload="" idx=1
|
||||
for part in "${@:3}"; do
|
||||
if ! response=$(upload_part_rest "$1" "$2" "$upload_id" "$idx" "$part" 2>&1); then
|
||||
for part in "${parts[@]}"; do
|
||||
if ! response=$(upload_part_rest "$bucket" "$key" "$upload_id" "$idx" "$part" 2>&1); then
|
||||
log 2 "error uploading part $idx: $response"
|
||||
return 1
|
||||
fi
|
||||
@@ -287,7 +318,7 @@ perform_multipart_upload_rest_variable_parts() {
|
||||
done
|
||||
log 5 "final payload: $parts_payload"
|
||||
|
||||
if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then
|
||||
if ! complete_multipart_upload_rest "$bucket" "$key" "$upload_id" "$parts_payload"; then
|
||||
log 2 "error completing multipart upload"
|
||||
return 1
|
||||
fi
|
||||
@@ -298,15 +329,120 @@ complete_multipart_upload_empty_upload_id() {
|
||||
if ! check_param_count_v2 "bucket, key, file" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" key="$2" file="$3"
|
||||
local response upload_id parts_payload
|
||||
if ! response=$(multipart_upload_rest_before_completion "$1" "$2" "$3" 2 2>&1); then
|
||||
|
||||
if ! response=$(multipart_upload_rest_before_completion "$bucket" "$key" "$file" 2 2>&1); then
|
||||
log 2 "error performing multipart upload before completion: $response"
|
||||
return 1
|
||||
fi
|
||||
read -r upload_id parts_payload <<< "$response"
|
||||
if ! complete_multipart_upload_rest_expect_error "$1" "$2" "" "$parts_payload" "" "404" "NoSuchUpload" "The specified upload does not exist"; then
|
||||
if ! complete_multipart_upload_rest_expect_error "$bucket" "$key" "" "$parts_payload" "" "404" "NoSuchUpload" "The specified upload does not exist"; then
|
||||
log 2 "error completing multipart upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
create_upload_part_copy_rest() {
|
||||
if ! check_param_count "create_upload_part_copy_rest" "bucket, key, >20MB file" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" key="$2" large_file="$3"
|
||||
local response upload_id file_name parts_payload i part_number result etag
|
||||
|
||||
if ! split_and_put_file "$bucket" "$key" "$large_file" 4; then
|
||||
log 2 "error splitting and putting file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! response=$(create_multipart_upload_rest "$bucket" "$key" "" "parse_upload_id" 2>&1); then
|
||||
log 2 "error creating upload and getting ID: $response"
|
||||
return 1
|
||||
fi
|
||||
upload_id="$response"
|
||||
|
||||
if ! response=$(get_file_name 2>&1); then
|
||||
log 2 "error getting file name: $response"
|
||||
return 1
|
||||
fi
|
||||
file_name="$response"
|
||||
|
||||
parts_payload=""
|
||||
for ((i=0; i<=3; i++)); do
|
||||
part_number=$((i+1))
|
||||
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" PART_NUMBER="$part_number" UPLOAD_ID="$upload_id" PART_LOCATION="${bucket}/${key}-$i" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/upload_part_copy.sh); then
|
||||
# shellcheck disable=SC2154
|
||||
log 2 "error uploading part $i: $result"
|
||||
return 1
|
||||
fi
|
||||
log 5 "result: $result"
|
||||
if [ "$result" != "200" ]; then
|
||||
log 2 "error uploading part $i: $(cat "$TEST_FILE_FOLDER/$file_name")"
|
||||
return 1
|
||||
fi
|
||||
if ! etag=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/$file_name" 2>&1); then
|
||||
log 2 "error retrieving etag: $etag"
|
||||
return 1
|
||||
fi
|
||||
parts_payload+="<Part><ETag>$etag</ETag><PartNumber>$part_number</PartNumber></Part>"
|
||||
done
|
||||
if ! complete_multipart_upload_rest "$bucket" "$key" "$upload_id" "$parts_payload"; then
|
||||
log 2 "error completing multipart upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
create_upload_finish_wrong_etag() {
|
||||
if ! check_param_count "create_upload_finish_wrong_etag" "bucket, key" 2 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" key="$2"
|
||||
local etag part_number response parts_payload result result_file error_file upload_id
|
||||
|
||||
if ! response=$(get_file_names 2 2>&1); then
|
||||
log 2 "error getting file names: $response"
|
||||
return 1
|
||||
fi
|
||||
read -r result_file error_file <<< "$response"
|
||||
|
||||
etag="gibberish"
|
||||
part_number=1
|
||||
if ! response=$(create_multipart_upload_rest "$bucket" "$key" "" "parse_upload_id" 2>&1); then
|
||||
log 2 "error creating upload and getting ID: $response"
|
||||
return 1
|
||||
fi
|
||||
upload_id="$response"
|
||||
|
||||
parts_payload="<Part><ETag>$etag</ETag><PartNumber>$part_number</PartNumber></Part>"
|
||||
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/$result_file" ./tests/rest_scripts/complete_multipart_upload.sh); then
|
||||
log 2 "error completing multipart upload: $result"
|
||||
return 1
|
||||
fi
|
||||
if [ "$result" != "400" ]; then
|
||||
log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/$result_file")"
|
||||
return 1
|
||||
fi
|
||||
if ! get_xml_data "$TEST_FILE_FOLDER/$result_file" "$TEST_FILE_FOLDER/$error_file"; then
|
||||
log 2 "error getting XML data"
|
||||
return 1
|
||||
fi
|
||||
if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "InvalidPart" "Code"; then
|
||||
log 2 "code mismatch"
|
||||
return 1
|
||||
fi
|
||||
if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "$upload_id" "UploadId"; then
|
||||
log 2 "upload ID mismatch"
|
||||
return 1
|
||||
fi
|
||||
if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "$part_number" "PartNumber"; then
|
||||
log 2 "part number mismatch"
|
||||
return 1
|
||||
fi
|
||||
if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "$etag" "ETag"; then
|
||||
log 2 "etag mismatch"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2026 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
multipart_upload_s3api_complete_from_bucket() {
|
||||
if ! check_param_count "multipart_upload_s3api_complete_from_bucket" "bucket, copy source, part count, upload ID" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" copy_source="$2" part_count="$3" upload_id="$4"
|
||||
local parts i response quoted_etag error
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= part_count; i++)); do
|
||||
if ! response=$(upload_part_copy "$bucket" "${copy_source}-copy" "$upload_id" "$copy_source" "$i" 2>&1); then
|
||||
log 2 "error uploading part $i: $response"
|
||||
return 1
|
||||
fi
|
||||
quoted_etag="$response"
|
||||
# UploadPartCopy etag is returned double quoted, so not necessary to quote it here
|
||||
parts+="{\"ETag\": $quoted_etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $part_count ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$bucket" --key "${copy_source}-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}' 2>&1); then
|
||||
log 2 "Error completing upload: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload_from_bucket() {
|
||||
if ! check_param_count "multipart_upload_from_bucket" "bucket, copy source, key, part count" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" copy_source="$2" key="$3" part_count="$4"
|
||||
local segments upload_id
|
||||
local -a segment_array
|
||||
|
||||
if ! segments=$(split_file "$key" "$part_count" 2>&1); then
|
||||
log 2 "error splitting file"
|
||||
return 1
|
||||
fi
|
||||
read -r -a segment_array <<< "$segments"
|
||||
|
||||
for ((i=0;i<part_count;i++)) {
|
||||
log 5 "key: '$key'"
|
||||
if ! put_object "s3api" "${segment_array[$i]}" "$bucket" "${copy_source}-$i"; then
|
||||
log 2 "error copying object"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
local response
|
||||
if ! response=$(create_multipart_upload_rest "$bucket" "${copy_source}-copy" "" "parse_upload_id" 2>&1); then
|
||||
log 2 "error running first multipart upload: $response"
|
||||
return 1
|
||||
fi
|
||||
upload_id="$response"
|
||||
|
||||
if ! multipart_upload_s3api_complete_from_bucket "$bucket" "$copy_source" "$part_count" "$upload_id"; then
|
||||
log 2 " error completing multipart upload from bucket"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload_from_bucket_range() {
|
||||
if ! check_param_count "multipart_upload_from_bucket_range" "bucket, copy source, key, part count, range" 5 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" copy_source="$2" key="$3" part_count="$4" range="$5"
|
||||
local segments parts quoted_etag upload_id
|
||||
local -a segment_array
|
||||
|
||||
if ! segments=$(split_file "$key" "$part_count" 2>&1); then
|
||||
log 2 "error splitting file"
|
||||
return 1
|
||||
fi
|
||||
read -r -a segment_array <<< "$segments"
|
||||
|
||||
for ((i=0;i<part_count;i++)) {
|
||||
log 5 "key: '$key', file info: '$(ls -l "${key}"-"$i")'"
|
||||
if ! put_object "s3api" "${segment_array[$i]}" "$bucket" "${copy_source}-$i"; then
|
||||
log 2 "error copying object"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
local response
|
||||
if ! response=$(create_multipart_upload_rest "$bucket" "${copy_source}-copy" "" "parse_upload_id" 2>&1); then
|
||||
log 2 "error running first multpart upload: $response"
|
||||
return 1
|
||||
fi
|
||||
upload_id="$response"
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= part_count; i++)); do
|
||||
if ! response=$(upload_part_copy_with_range "$bucket" "${copy_source}-copy" "$upload_id" "$copy_source" "$i" "$range" 2>&1); then
|
||||
# shellcheck disable=SC2154
|
||||
log 2 "error uploading part $i: '$response'"
|
||||
return 1
|
||||
fi
|
||||
quoted_etag="$response"
|
||||
# UploadPartCopy etag is returned double quoted, so not necessary to quote it here
|
||||
parts+="{\"ETag\": $quoted_etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}'); then
|
||||
log 2 "Error completing upload: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload_custom() {
|
||||
if ! check_param_count_gt "bucket, key, file, part count, optional additional parameters" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" key="$2" file="$3" part_count="$4" params=("${@:5}")
|
||||
|
||||
# shellcheck disable=SC2086 disable=SC2048
|
||||
if ! multipart_upload_before_completion_custom "$bucket" "$key" "$file" "$part_count" "${params[@]}"; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log 5 "upload ID: $upload_id, parts: $parts"
|
||||
if ! complete_multipart_upload "$bucket" "$key" "$upload_id" "$parts"; then
|
||||
log 2 "Error completing upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload() {
|
||||
if ! check_param_count "multipart_upload" "bucket, key, file, part count" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" key="$2" file="$3" part_count="$4"
|
||||
|
||||
if ! multipart_upload_before_completion "$bucket" "$key" "$file" "$part_count"; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! complete_multipart_upload "$bucket" "$key" "$upload_id" "$parts"; then
|
||||
log 2 "Error completing upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# perform a multi-part upload
|
||||
# params: bucket, key, source file location, number of parts
|
||||
# return 0 for success, 1 for failure
|
||||
multipart_upload_with_params() {
|
||||
if ! check_param_count "multipart_upload_with_params" "bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging" 10 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1"
|
||||
local key="$2"
|
||||
local file="$3"
|
||||
local part_count="$4"
|
||||
local content_type="$5"
|
||||
local metadata="$6"
|
||||
local hold_status="$7"
|
||||
local lock_mode="$8"
|
||||
local retain_until_date="$9"
|
||||
local tagging="${10}"
|
||||
|
||||
if ! multipart_upload_before_completion_with_params "$bucket" "$key" "$file" "$part_count" "$content_type" \
|
||||
"$metadata" "$hold_status" "$lock_mode" "$retain_until_date" "$tagging"; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
log 5 "Upload parts: $parts"
|
||||
|
||||
if ! complete_multipart_upload "$bucket" "$key" "$upload_id" "$parts"; then
|
||||
log 2 "Error completing upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2026 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
setup_multipart_upload_with_params() {
|
||||
if ! check_param_count "setup_multipart_upload_with_params" "bucket, key" 2 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" key="$2"
|
||||
local os_name later_seconds now_epoch later_epoch later result
|
||||
|
||||
os_name="$(uname)"
|
||||
if [ "$DIRECT" == "true" ]; then
|
||||
later_seconds="40"
|
||||
else
|
||||
later_seconds="20"
|
||||
fi
|
||||
|
||||
now_epoch=$(date +%s)
|
||||
later_epoch=$((now_epoch + later_seconds))
|
||||
if [[ "$os_name" == "Darwin" ]]; then
|
||||
later=$(date -r "$later_epoch" -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
else
|
||||
later=$(date -u -d "@$later_epoch" +"%Y-%m-%dT%H:%M:%SZ")
|
||||
fi
|
||||
log 5 "later in function: $later"
|
||||
|
||||
if ! result=$(dd if=/dev/urandom of="${TEST_FILE_FOLDER}/${key}" bs=20971520 count=1 2>&1); then
|
||||
log 2 "error creating large file: $result"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! setup_bucket_object_lock_enabled_v2 "$bucket"; then
|
||||
log 2 "error creating bucket: $bucket"
|
||||
return 1
|
||||
fi
|
||||
log 5 "later in function: $later"
|
||||
echo "$later"
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2026 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
run_and_verify_multipart_upload_with_valid_range() {
|
||||
if ! check_param_count "run_and_verify_multipart_upload_with_valid_range" "bucket, key, full 5MB file path" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" key="$2" large_file="$3"
|
||||
local range_max=$((5*1024*1024-1)) object_size
|
||||
|
||||
if ! multipart_upload_from_bucket_range "$bucket" "$key" "$large_file" 4 "bytes=0-$range_max"; then
|
||||
log 2 "error with multipart upload"
|
||||
return 1
|
||||
fi
|
||||
if ! get_object "s3api" "$bucket" "${key}-copy" "${large_file}-copy"; then
|
||||
log 2 "error getting object"
|
||||
return 1
|
||||
fi
|
||||
if [[ $(uname) == 'Darwin' ]]; then
|
||||
object_size=$(stat -f%z "${large_file}-copy")
|
||||
else
|
||||
object_size=$(stat --format=%s "${large_file}-copy")
|
||||
fi
|
||||
if [[ object_size -ne $((range_max*4+4)) ]]; then
|
||||
log 2 "object size mismatch ($object_size, $((range_max*4+4)))"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
@@ -14,13 +14,38 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
get_and_verify_metadata() {
|
||||
if [ $# -ne 7 ]; then
|
||||
log 2 "'get_and_verify_metadata' requires bucket file, expected content type,
|
||||
expected metadata key, expected metadata val, expected hold status, expected retention mode, expected retention date"
|
||||
get_and_verify_metadata_value() {
|
||||
if ! check_param_count_v2 "data file, expected metadata key, expected metadata val" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
if ! head_object "s3api" "$BUCKET_ONE_NAME" "$1"; then
|
||||
local data_file="$1" expected_metadata_key="$2" expected_metadata_val="$3"
|
||||
local response value
|
||||
|
||||
if ! response=$(jq -r "$expected_metadata_key" - <<< "$data_file" 2>&1); then
|
||||
log 2 "error retrieving content type: $response"
|
||||
return 1
|
||||
fi
|
||||
value="$response"
|
||||
if [[ "$value" != "$expected_metadata_val" ]]; then
|
||||
log 2 "'$expected_metadata_key' mismatch (expected '$expected_metadata_val', actual '$value')"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
get_and_verify_metadata() {
|
||||
if ! check_param_count_v2 "bucket file, expected content type, expected metadata key, expected metadata val, expected hold status, expected retention mode, expected retention date" 7 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket_file="$1"
|
||||
local expected_content_type="$2"
|
||||
local expected_metadata_key="$3"
|
||||
local expected_metadata_val="$4"
|
||||
local expected_hold_status="$5"
|
||||
local expected_retention_mode="$6"
|
||||
local expected_retention_date="$7"
|
||||
|
||||
if ! head_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file"; then
|
||||
log 2 "error retrieving metadata"
|
||||
return 1
|
||||
fi
|
||||
@@ -28,45 +53,24 @@ get_and_verify_metadata() {
|
||||
raw_metadata=$(echo "$metadata" | grep -v "InsecureRequestWarning")
|
||||
log 5 "raw metadata: $raw_metadata"
|
||||
|
||||
if ! content_type=$(echo "$raw_metadata" | jq -r ".ContentType" 2>&1); then
|
||||
log 2 "error retrieving content type: $content_type"
|
||||
if ! get_and_verify_metadata_value "$raw_metadata" ".ContentType" "$expected_content_type"; then
|
||||
log 2 "error verifying .ContentType"
|
||||
return 1
|
||||
fi
|
||||
if [[ $content_type != "$2" ]]; then
|
||||
log 2 "content type mismatch ($content_type, $2)"
|
||||
if ! get_and_verify_metadata_value "$raw_metadata" ".Metadata.${expected_metadata_key}" "$expected_metadata_val"; then
|
||||
log 2 "error verifying .Metadata.${expected_metadata_key}"
|
||||
return 1
|
||||
fi
|
||||
log 5 "metadata key: $3"
|
||||
if ! meta_val=$(echo "$raw_metadata" | jq -r ".Metadata.$3" 2>&1); then
|
||||
log 2 "error retrieving metadata val: $meta_val"
|
||||
if ! get_and_verify_metadata_value "$raw_metadata" ".ObjectLockLegalHoldStatus" "$expected_hold_status"; then
|
||||
log 2 "error verifying .ObjectLockLegalHoldStatus"
|
||||
return 1
|
||||
fi
|
||||
if [[ $meta_val != "$4" ]]; then
|
||||
log 2 "metadata val mismatch ($meta_val, $4)"
|
||||
if ! get_and_verify_metadata_value "$raw_metadata" ".ObjectLockMode" "$expected_retention_mode"; then
|
||||
log 2 "error verifying .ObjectLockLegalHoldStatus"
|
||||
return 1
|
||||
fi
|
||||
if ! hold_status=$(echo "$raw_metadata" | jq -r ".ObjectLockLegalHoldStatus" 2>&1); then
|
||||
log 2 "error retrieving hold status: $hold_status"
|
||||
return 1
|
||||
fi
|
||||
if [[ $hold_status != "$5" ]]; then
|
||||
log 2 "hold status mismatch ($hold_status, $5)"
|
||||
return 1
|
||||
fi
|
||||
if ! retention_mode=$(echo "$raw_metadata" | jq -r ".ObjectLockMode" 2>&1); then
|
||||
log 2 "error retrieving retention mode: $retention_mode"
|
||||
return 1
|
||||
fi
|
||||
if [[ $retention_mode != "$6" ]]; then
|
||||
log 2 "retention mode mismatch ($retention_mode, $6)"
|
||||
return 1
|
||||
fi
|
||||
if ! retain_until_date=$(echo "$raw_metadata" | jq -r ".ObjectLockRetainUntilDate" 2>&1); then
|
||||
log 2 "error retrieving retain until date: $retain_until_date"
|
||||
return 1
|
||||
fi
|
||||
if [[ $retain_until_date != "$7"* ]]; then
|
||||
log 2"retention date mismatch ($retain_until_date, $7)"
|
||||
if ! get_and_verify_metadata_value "$raw_metadata" ".ObjectLockRetainUntilDate" "$expected_retention_date"; then
|
||||
log 2 "error verifying .ObjectLockRetainUntilDate"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
# Copyright 2026 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
check_redirect_response() {
|
||||
if ! check_param_count_v2 "URL, redirect URL, protocol" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
local url="$1" redirect_url="$2" protocol="$3"
|
||||
local response response_file
|
||||
|
||||
if ! response=$(curl -ksi "${protocol}://${url}" 2>&1); then
|
||||
log 2 "error sending curl message: $response"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$response" != *"HTTP/1.1 301"* ]]; then
|
||||
log 2 "expected 301 response (response: '$response')"
|
||||
return 1
|
||||
fi
|
||||
if ! response_file=$(get_file_name 2>&1); then
|
||||
log 2 "error getting file name: $response_file"
|
||||
return 1
|
||||
fi
|
||||
echo "$response" > "$TEST_FILE_FOLDER/$response_file"
|
||||
if ! check_header_key_and_value "$TEST_FILE_FOLDER/$response_file" "Location" "${protocol}://$redirect_url/"; then
|
||||
log 2 "error checking redirect header key and value (data: '$response')"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
@@ -51,4 +51,39 @@ put_public_bucket_policy() {
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
create_website_with_random_string_and_add_permissions() {
|
||||
if ! check_param_count_v2 "bucket name" 1 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket_name="$1"
|
||||
local response random_string policy_file
|
||||
|
||||
if ! response=$(create_website_with_random_string "$bucket_name" 2>&1); then
|
||||
log 2 "error creating website: $response"
|
||||
return 1
|
||||
fi
|
||||
read -r _ random_string <<< "$response"
|
||||
log 5 "random string: $random_string"
|
||||
|
||||
if [ "$DIRECT" == "true" ]; then
|
||||
if ! put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false" > /dev/null 2>&1; then
|
||||
log 2 "error putting public access block"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! response=$(setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" 2>&1); then
|
||||
log 2 "error setting up policy: $response"
|
||||
return 1
|
||||
fi
|
||||
policy_file="$response"
|
||||
|
||||
if ! put_bucket_policy_rest "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"; then
|
||||
log 2 "error putting bucket policy"
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$random_string"
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
# under the License.
|
||||
|
||||
source ./tests/drivers/create_bucket/create_bucket_rest.sh
|
||||
source ./tests/drivers/put_bucket_policy/put_bucket_policy_rest.sh
|
||||
source ./tests/drivers/string.sh
|
||||
|
||||
create_website_with_random_string() {
|
||||
@@ -37,4 +38,25 @@ create_website_with_random_string() {
|
||||
fi
|
||||
printf '%s\n' "$file_name $test_string"
|
||||
return 0
|
||||
}
|
||||
|
||||
put_redirect() {
|
||||
if ! check_param_count_ge_le "bucket name, redirect URL, protocol (optional)" 2 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket_name="$1" redirect_url="$2" protocol="$3"
|
||||
local json_payload
|
||||
|
||||
if [[ "$protocol" == "https" ]] || [[ "$protocol" == "http" ]]; then
|
||||
json_payload="{\"RedirectAllRequestsTo\":{\"HostName\":\"$redirect_url\",\"Protocol\":\"$protocol\"}}"
|
||||
else
|
||||
json_payload="{\"RedirectAllRequestsTo\":{\"HostName\":\"$redirect_url\"}}"
|
||||
fi
|
||||
|
||||
if ! response=$(send_rest_go_command "200" "-commandType" "putBucketWebsiteConfiguration" "-bucketName" "$bucket_name" \
|
||||
"-websiteConfiguration" "$json_payload" 2>&1); then
|
||||
log 2 "error putting website configuration: $response"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2026 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
split_and_put_file() {
|
||||
if ! check_param_count_v2 "bucket, key, copy source, part count" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
local bucket="$1" key="$2" copy_source="$3" part_count="$4"
|
||||
local file_parts idx
|
||||
local -a part_array
|
||||
|
||||
if ! file_parts=$(split_file "$copy_source" "$part_count" 2>&1); then
|
||||
log 2 "error splitting file: $file_parts"
|
||||
return 1
|
||||
fi
|
||||
read -r -a part_array <<< "$file_parts"
|
||||
|
||||
for ((idx=0;idx<part_count;idx++)) {
|
||||
log 5 "key: $key, file info: $(ls -l "${part_array[$idx]}")"
|
||||
if ! put_object "s3api" "${part_array[$idx]}" "$bucket" "${key}-${idx}"; then
|
||||
log 2 "error copying object"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -15,6 +15,7 @@
|
||||
# under the License.
|
||||
|
||||
source ./tests/commands/command.sh
|
||||
source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh
|
||||
source ./tests/drivers/list_objects/list_objects_rest.sh
|
||||
source ./tests/drivers/xml.sh
|
||||
|
||||
|
||||
+10
-3
@@ -644,9 +644,16 @@ if ! install_managed_libraries "$os"; then
|
||||
fi
|
||||
|
||||
if [ -z "$DOWNLOAD_ONLY_FOLDER" ]; then
|
||||
if ! response=$(go get -v -t -d ./... 2>&1); then
|
||||
printf "error installing go dependencies: %s\n" "$response"
|
||||
exit 1
|
||||
if [ -n "$INSTALL_ONLY_FOLDER" ]; then
|
||||
if ! response=$(GOPROXY=off go mod download 2>&1); then
|
||||
printf "error installing go dependencies: %s\n" "$response"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if ! response=$(go mod download 2>&1); then
|
||||
printf "error installing go dependencies: %s\n" "$response"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ load ./bats-assert/load
|
||||
source ./tests/commands/get_object.sh
|
||||
source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh
|
||||
source ./tests/drivers/create_bucket/create_bucket_rest.sh
|
||||
source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh
|
||||
source ./tests/setup.sh
|
||||
|
||||
# tags: curl,GetObject,HeadObject,range
|
||||
|
||||
@@ -19,6 +19,7 @@ load ./bats-assert/load
|
||||
|
||||
source ./tests/setup.sh
|
||||
source ./tests/drivers/create_bucket/create_bucket_rest.sh
|
||||
source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh
|
||||
source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh
|
||||
source ./tests/drivers/get_object_attributes/get_object_attributes_rest.sh
|
||||
source ./tests/drivers/string.sh
|
||||
|
||||
@@ -18,6 +18,7 @@ load ./bats-support/load
|
||||
load ./bats-assert/load
|
||||
|
||||
source ./tests/drivers/create_bucket/create_bucket_rest.sh
|
||||
source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh
|
||||
source ./tests/setup.sh
|
||||
|
||||
@test "REST - ListParts - invalid part number marker" {
|
||||
|
||||
@@ -21,8 +21,10 @@ source ./tests/setup.sh
|
||||
source ./tests/drivers/file.sh
|
||||
source ./tests/drivers/rest.sh
|
||||
source ./tests/drivers/create_bucket/create_bucket_rest.sh
|
||||
source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh
|
||||
source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh
|
||||
source ./tests/drivers/list_buckets/list_buckets_rest.sh
|
||||
source ./tests/drivers/put_object/put_object_s3api.sh
|
||||
source ./tests/drivers/upload_part/upload_part_rest.sh
|
||||
|
||||
# tags: curl,multipart,CreateMultipartUpload,AbortMultipartUpload
|
||||
|
||||
@@ -22,6 +22,7 @@ source ./tests/drivers/create_bucket/create_bucket_rest.sh
|
||||
source ./tests/drivers/get_bucket_website/get_bucket_website_rest.sh
|
||||
source ./tests/drivers/put_bucket_website/put_bucket_website_rest.sh
|
||||
source ./tests/drivers/cloudfront.sh
|
||||
source ./tests/drivers/http.sh
|
||||
source ./tests/drivers/string.sh
|
||||
source ./tests/setup_common.sh
|
||||
|
||||
@@ -35,6 +36,10 @@ setup() {
|
||||
setup_versitygw_for_website() {
|
||||
optional_params=("$@")
|
||||
|
||||
if [ "$DIRECT" == "true" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
run setup_versitygw "${optional_params[@]}"
|
||||
assert_success
|
||||
read -r process_id process_id_two <<< "$output"
|
||||
@@ -42,6 +47,7 @@ setup_versitygw_for_website() {
|
||||
if [ -n "$process_id_two" ]; then
|
||||
export VERSITYGW_PID_2="$process_id_two"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
teardown() {
|
||||
@@ -154,20 +160,39 @@ teardown() {
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "REST - GetBucketWebsite - no HTTPS" {
|
||||
local bucket_name random_string
|
||||
@test "REST - GetBucketWebsite - redirect" {
|
||||
local bucket_one_name bucket_two_name random_string protocol="http"
|
||||
|
||||
setup_versitygw_for_website "--website-no-tls"
|
||||
|
||||
run setup_bucket_v3 "$BUCKET_ONE_NAME"
|
||||
run setup_buckets_v3 "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME"
|
||||
assert_success
|
||||
bucket_name="$output"
|
||||
read -r bucket_one_name bucket_two_name <<< "$output"
|
||||
|
||||
run create_website_with_random_string_and_add_permissions "$bucket_name"
|
||||
run create_website_with_random_string_and_add_permissions "$bucket_one_name"
|
||||
assert_success
|
||||
read -r random_string <<< "$output"
|
||||
log 5 "random string after output: $random_string"
|
||||
|
||||
run curl -ks "http://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}"
|
||||
if [ "$DIRECT" == "true" ]; then
|
||||
domain_name="${bucket_one_name}.s3-website.${AWS_REGION}.amazonaws.com"
|
||||
else
|
||||
domain_name="${bucket_one_name}.${WEBSITE_DOMAIN}${WEBSITE}"
|
||||
fi
|
||||
|
||||
run put_redirect "$bucket_two_name" "${domain_name}" "$protocol"
|
||||
assert_success
|
||||
|
||||
if [ "$DIRECT" == "true" ]; then
|
||||
domain_two_name="${bucket_two_name}.s3-website.${AWS_REGION}.amazonaws.com"
|
||||
else
|
||||
domain_two_name="${bucket_two_name}.${WEBSITE_DOMAIN}${WEBSITE}"
|
||||
fi
|
||||
|
||||
run check_redirect_response "$domain_two_name" "$domain_name" "$protocol"
|
||||
assert_success
|
||||
|
||||
run curl -ksL "${protocol}://${domain_two_name}"
|
||||
assert_success
|
||||
assert_output "$random_string"
|
||||
}
|
||||
|
||||
@@ -23,13 +23,16 @@ source ./tests/commands/get_object.sh
|
||||
source ./tests/commands/list_multipart_uploads.sh
|
||||
source ./tests/commands/put_object.sh
|
||||
source ./tests/drivers/file.sh
|
||||
source ./tests/drivers/head_object/head_object_s3api.sh
|
||||
source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_s3api.sh
|
||||
source ./tests/drivers/create_bucket/create_bucket_rest.sh
|
||||
source ./tests/drivers/create_bucket/create_bucket_s3api.sh
|
||||
source ./tests/drivers/get_object/get_object_s3api.sh
|
||||
source ./tests/drivers/get_object_legal_hold/get_object_legal_hold.sh
|
||||
source ./tests/drivers/get_object_tagging/get_object_tagging.sh
|
||||
source ./tests/drivers/head_object/head_object_s3api.sh
|
||||
source ./tests/drivers/list_multipart_uploads/list_multipart_uploads_s3api.sh
|
||||
source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh
|
||||
source ./tests/util/util_multipart.sh
|
||||
source ./tests/drivers/put_object/put_object_s3api.sh
|
||||
source ./tests/util/util_multipart_abort.sh
|
||||
source ./tests/util/util_multipart_before_completion.sh
|
||||
|
||||
@@ -91,6 +94,8 @@ export RUN_USERS=true
|
||||
"$expected_tag_key=$expected_tag_val"
|
||||
assert_success
|
||||
|
||||
log 5 "later: $later"
|
||||
later="${later/Z/+00:00}"
|
||||
run get_and_verify_metadata "$bucket_file" "$expected_content_type" "$expected_meta_key" "$expected_meta_val" \
|
||||
"$expected_hold_status" "$expected_retention_mode" "$later"
|
||||
assert_success
|
||||
|
||||
@@ -22,7 +22,6 @@ source ./tests/setup.sh
|
||||
source ./tests/test_s3api_policy_bucket.sh
|
||||
source ./tests/test_s3api_policy_multipart.sh
|
||||
source ./tests/test_s3api_policy_object.sh
|
||||
source ./tests/util/util_multipart.sh
|
||||
source ./tests/util/util_multipart_abort.sh
|
||||
source ./tests/util/util_multipart_before_completion.sh
|
||||
source ./tests/util/util_policy.sh
|
||||
|
||||
@@ -1,355 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2024 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
source ./tests/commands/put_object.sh
|
||||
source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh
|
||||
|
||||
multipart_upload_s3api_complete_from_bucket() {
|
||||
if ! check_param_count "multipart_upload_s3api_complete_from_bucket" "bucket, copy source, part count" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
parts="["
|
||||
for ((i = 1; i <= $3; i++)); do
|
||||
# shellcheck disable=SC2154
|
||||
if ! upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i"; then
|
||||
log 2 "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2154
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $3 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}' 2>&1); then
|
||||
log 2 "Error completing upload: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload_from_bucket() {
|
||||
if ! check_param_count "multipart_upload_from_bucket" "bucket, copy source, key, part count" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! segments=$(split_file "$3" "$4" 2>&1); then
|
||||
log 2 "error splitting file"
|
||||
return 1
|
||||
fi
|
||||
read -r -a segment_array <<< "$segments"
|
||||
|
||||
for ((i=0;i<$4;i++)) {
|
||||
log 5 "key: $3"
|
||||
if ! put_object "s3api" "${segment_array[$i]}" "$1" "$2-$i"; then
|
||||
log 2 "error copying object"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
local response
|
||||
if ! response=$(create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id" 2>&1); then
|
||||
log 2 "error running first multipart upload: $response"
|
||||
return 1
|
||||
fi
|
||||
upload_id="$response"
|
||||
|
||||
if ! multipart_upload_s3api_complete_from_bucket "$1" "$2" "$4"; then
|
||||
log 2 " error completing multipart upload from bucket"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
split_and_put_file() {
|
||||
if ! check_param_count "split_and_put_file" "bucket, key, copy source, part count" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
if ! file_parts=$(split_file "$3" "$4" 2>&1); then
|
||||
log 2 "error splitting file: $file_parts"
|
||||
return 1
|
||||
fi
|
||||
read -r -a part_array <<< "$file_parts"
|
||||
|
||||
for ((i=0;i<$4;i++)) {
|
||||
log 5 "key: $2, file info: $(ls -l "${part_array[$i]}")"
|
||||
if ! put_object "s3api" "${part_array[$i]}" "$1" "$2-$i"; then
|
||||
log 2 "error copying object"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload_from_bucket_range() {
|
||||
if ! check_param_count "multipart_upload_from_bucket_range" "bucket, copy source, key, part count, range" 5 $#; then
|
||||
return 1
|
||||
fi
|
||||
if ! segments=$(split_file "$3" "$4" 2>&1); then
|
||||
log 2 "error splitting file"
|
||||
return 1
|
||||
fi
|
||||
read -r -a segment_array <<< "$segments"
|
||||
|
||||
for ((i=0;i<$4;i++)) {
|
||||
log 5 "key: $3, file info: $(ls -l "$3"-"$i")"
|
||||
if ! put_object "s3api" "${segment_array[$i]}" "$1" "$2-$i"; then
|
||||
log 2 "error copying object"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
local response
|
||||
if ! response=$(create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id" 2>&1); then
|
||||
log 2 "error running first multpart upload: $response"
|
||||
return 1
|
||||
fi
|
||||
upload_id="$response"
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
if ! upload_part_copy_with_range "$1" "$2-copy" "$upload_id" "$2" "$i" "$5"; then
|
||||
# shellcheck disable=SC2154
|
||||
log 2 "error uploading part $i: $upload_part_copy_error"
|
||||
return 1
|
||||
fi
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}'); then
|
||||
log 2 "Error completing upload: $error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload_custom() {
|
||||
if ! check_param_count_gt "bucket, key, file, part count, optional additional parameters" 4 $$; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086 disable=SC2048
|
||||
if ! multipart_upload_before_completion_custom "$1" "$2" "$3" "$4" ${*:5}; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log 5 "upload ID: $upload_id, parts: $parts"
|
||||
if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then
|
||||
log 2 "Error completing upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload() {
|
||||
if ! check_param_count "multipart_upload" "bucket, key, file, part count" 4 $#; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then
|
||||
log 2 "Error completing upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# perform a multi-part upload
|
||||
# params: bucket, key, source file location, number of parts
|
||||
# return 0 for success, 1 for failure
|
||||
multipart_upload_with_params() {
|
||||
if ! check_param_count "multipart_upload_with_params" "bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging" 10 $#; then
|
||||
return 1
|
||||
fi
|
||||
log 5 "1: $1, 2: $2, 3: $3, 4: $4, 5: $5, 6: $6, 7: $7, 8: $8, 9: $9, 10: ${10}"
|
||||
|
||||
if ! multipart_upload_before_completion_with_params "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}"; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
log 5 "Upload parts: $parts"
|
||||
|
||||
if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then
|
||||
log 2 "Error completing upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
run_and_verify_multipart_upload_with_valid_range() {
|
||||
if ! check_param_count "run_and_verify_multipart_upload_with_valid_range" "bucket, key, 5MB file" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
range_max=$((5*1024*1024-1))
|
||||
if ! multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-$range_max"; then
|
||||
log 2 "error with multipart upload"
|
||||
return 1
|
||||
fi
|
||||
if ! get_object "s3api" "$1" "$2-copy" "$3-copy"; then
|
||||
log 2 "error getting object"
|
||||
return 1
|
||||
fi
|
||||
if [[ $(uname) == 'Darwin' ]]; then
|
||||
object_size=$(stat -f%z "$3-copy")
|
||||
else
|
||||
object_size=$(stat --format=%s "$3-copy")
|
||||
fi
|
||||
if [[ object_size -ne $((range_max*4+4)) ]]; then
|
||||
log 2 "object size mismatch ($object_size, $((range_max*4+4)))"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
create_upload_part_copy_rest() {
|
||||
if ! check_param_count "create_upload_part_copy_rest" "bucket, key, >20MB file" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
if ! split_and_put_file "$1" "$2" "$3" 4; then
|
||||
log 2 "error splitting and putting file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local response
|
||||
if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then
|
||||
log 2 "error creating upload and getting ID: $response"
|
||||
return 1
|
||||
fi
|
||||
upload_id="$response"
|
||||
|
||||
parts_payload=""
|
||||
for ((i=0; i<=3; i++)); do
|
||||
part_number=$((i+1))
|
||||
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="$part_number" UPLOAD_ID="$upload_id" PART_LOCATION="$1/$2-$i" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part_copy.sh); then
|
||||
# shellcheck disable=SC2154
|
||||
log 2 "error uploading part $i: $result"
|
||||
return 1
|
||||
fi
|
||||
log 5 "result: $result"
|
||||
if [ "$result" != "200" ]; then
|
||||
log 2 "error uploading part $i: $(cat "$TEST_FILE_FOLDER/response.txt")"
|
||||
return 1
|
||||
fi
|
||||
if ! etag=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/response.txt" 2>&1); then
|
||||
log 2 "error retrieving etag: $etag"
|
||||
return 1
|
||||
fi
|
||||
parts_payload+="<Part><ETag>$etag</ETag><PartNumber>$part_number</PartNumber></Part>"
|
||||
done
|
||||
if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then
|
||||
log 2 "error completing multipart upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
create_upload_finish_wrong_etag() {
|
||||
if ! check_param_count "create_upload_finish_wrong_etag" "bucket, key" 2 $#; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
etag="gibberish"
|
||||
part_number=1
|
||||
if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then
|
||||
log 2 "error creating upload and getting ID"
|
||||
return 1
|
||||
fi
|
||||
parts_payload="<Part><ETag>$etag</ETag><PartNumber>$part_number</PartNumber></Part>"
|
||||
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then
|
||||
log 2 "error completing multipart upload: $result"
|
||||
return 1
|
||||
fi
|
||||
if [ "$result" != "400" ]; then
|
||||
log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")"
|
||||
return 1
|
||||
fi
|
||||
if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error.txt"; then
|
||||
log 2 "error getting XML data"
|
||||
return 1
|
||||
fi
|
||||
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "InvalidPart" "Code"; then
|
||||
log 2 "code mismatch"
|
||||
return 1
|
||||
fi
|
||||
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "$upload_id" "UploadId"; then
|
||||
log 2 "upload ID mismatch"
|
||||
return 1
|
||||
fi
|
||||
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "$part_number" "PartNumber"; then
|
||||
log 2 "part number mismatch"
|
||||
return 1
|
||||
fi
|
||||
if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "$etag" "ETag"; then
|
||||
log 2 "etag mismatch"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
setup_multipart_upload_with_params() {
|
||||
if ! check_param_count "setup_multipart_upload_with_params" "bucket, key" 2 $#; then
|
||||
return 1
|
||||
fi
|
||||
os_name="$(uname)"
|
||||
if [ "$DIRECT" == "true" ]; then
|
||||
later_seconds="40"
|
||||
else
|
||||
later_seconds="20"
|
||||
fi
|
||||
if [[ "$os_name" == "Darwin" ]]; then
|
||||
now=$(date -u +"%Y-%m-%dT%H:%M:%S")
|
||||
later=$(date -j -v "+${later_seconds}S" -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S")
|
||||
else
|
||||
now=$(date +"%Y-%m-%dT%H:%M:%S")
|
||||
later=$(date -d "$now $later_seconds seconds" +"%Y-%m-%dT%H:%M:%S")
|
||||
fi
|
||||
log 5 "later in function: $later"
|
||||
|
||||
if ! create_test_files "$2"; then
|
||||
log 2 "error creating test file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! result=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER/$2" bs=20M count=1 2>&1); then
|
||||
log 2 "error creating large file: $result"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"; then
|
||||
log 2 "error cleaning up bucket"
|
||||
return 1
|
||||
fi
|
||||
# in static bucket config, bucket will still exist
|
||||
if ! bucket_exists "$BUCKET_ONE_NAME"; then
|
||||
if ! create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"; then
|
||||
log 2 "error creating bucket with object lock enabled"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
log 5 "later in function: $later"
|
||||
echo "$later"
|
||||
return 0
|
||||
}
|
||||
@@ -339,7 +339,6 @@ multipart_upload_before_completion_custom() {
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
export parts
|
||||
}
|
||||
|
||||
@@ -347,14 +346,16 @@ multipart_upload_range_too_large() {
|
||||
if ! check_param_count_v2 "bucket, key, file location" 3 $#; then
|
||||
return 1
|
||||
fi
|
||||
if multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-1000000000"; then
|
||||
log 2 "multipart upload succeeded despite overly large range"
|
||||
local bucket="$1" key="$2" file_location="$3"
|
||||
local response
|
||||
|
||||
if response=$(multipart_upload_from_bucket_range "$bucket" "$key" "$file_location" 4 "bytes=0-1000000000" 2>&1); then
|
||||
log 2 "multipart upload succeeded despite overly large range: $response"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2154
|
||||
log 5 "error: $upload_part_copy_error"
|
||||
if [[ $upload_part_copy_error != *"Range specified is not valid"* ]] && [[ $upload_part_copy_error != *"InvalidRange"* ]]; then
|
||||
log 2 "unexpected error: $upload_part_copy_error"
|
||||
if [[ "$response" != *"Range specified is not valid"* ]] && [[ "$response" != *"InvalidRange"* ]]; then
|
||||
log 2 "unexpected error: $response"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
source ./tests/util/util_multipart.sh
|
||||
source ./tests/util/util_versioning.sh
|
||||
source ./tests/logger.sh
|
||||
source ./tests/commands/abort_multipart_upload.sh
|
||||
|
||||
Reference in New Issue
Block a user