mirror of
https://github.com/versity/versitygw.git
synced 2026-01-07 04:06:23 +00:00
Merge pull request #996 from versity/test/rest_list_objects_v1_2
Test - REST ListObjects v1 multi page
This commit is contained in:
Ben McClelland
GitHub
@@ -34,30 +34,10 @@ list_object_versions_rest() {
|
||||
log 2 "'list_object_versions_rest' requires bucket name"
|
||||
return 1
|
||||
fi
|
||||
generate_hash_for_payload ""
|
||||
|
||||
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
|
||||
# shellcheck disable=SC2154
|
||||
canonical_request="GET
|
||||
/$1
|
||||
versions=
|
||||
host:${AWS_ENDPOINT_URL#*//}
|
||||
x-amz-content-sha256:$payload_hash
|
||||
x-amz-date:$current_date_time
|
||||
|
||||
host;x-amz-content-sha256;x-amz-date
|
||||
$payload_hash"
|
||||
|
||||
if ! generate_sts_string "$current_date_time" "$canonical_request"; then
|
||||
log 2 "error generating sts string"
|
||||
log 5 "list object versions REST"
|
||||
if ! result=$(BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object_versions.txt" ./tests/rest_scripts/list_object_versions.sh); then
|
||||
log 2 "error listing object versions: $result"
|
||||
return 1
|
||||
fi
|
||||
|
||||
get_signature
|
||||
# shellcheck disable=SC2034,SC2154
|
||||
reply=$(send_command curl -ks "$AWS_ENDPOINT_URL/$1?versions" \
|
||||
-H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
|
||||
-H "x-amz-content-sha256: $payload_hash" \
|
||||
-H "x-amz-date: $current_date_time" \
|
||||
-o "$TEST_FILE_FOLDER/object_versions.txt" 2>&1)
|
||||
return 0
|
||||
}
|
||||
45
tests/rest_scripts/list_object_versions.sh
Executable file
45
tests/rest_scripts/list_object_versions.sh
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2024 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
source ./tests/rest_scripts/rest.sh
|
||||
|
||||
# Fields
|
||||
|
||||
# shellcheck disable=SC2153
|
||||
bucket_name="$BUCKET_NAME"
|
||||
|
||||
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
|
||||
|
||||
# shellcheck disable=SC2034
|
||||
canonical_request="GET
|
||||
/$bucket_name
|
||||
versions=
|
||||
host:$host
|
||||
x-amz-content-sha256:UNSIGNED-PAYLOAD
|
||||
x-amz-date:$current_date_time
|
||||
|
||||
host;x-amz-content-sha256;x-amz-date
|
||||
UNSIGNED-PAYLOAD"
|
||||
|
||||
create_canonical_hash_sts_and_signature
|
||||
|
||||
curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name?versions"
|
||||
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
|
||||
-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
|
||||
-H "\"x-amz-date: $current_date_time\""
|
||||
-o "$OUTPUT_FILE")
|
||||
# shellcheck disable=SC2154
|
||||
eval "${curl_command[*]}" 2>&1
|
||||
@@ -23,6 +23,8 @@ bucket_name="$BUCKET_NAME"
|
||||
version_two="${VERSION_TWO:-FALSE}"
|
||||
max_keys="${MAX_KEYS:-0}"
|
||||
# shellcheck disable=SC2153
|
||||
marker="$MARKER"
|
||||
# shellcheck disable=SC2153
|
||||
if [ "$CONTINUATION_TOKEN" != "" ]; then
|
||||
continuation_token=$(jq -rn --arg token "$CONTINUATION_TOKEN" '$token | @uri')
|
||||
fi
|
||||
@@ -34,6 +36,9 @@ canonical_request="GET
|
||||
/$bucket_name
|
||||
"
|
||||
|
||||
if [ "$MARKER" != "" ]; then
|
||||
add_parameter "canonical_request" "marker=$marker"
|
||||
fi
|
||||
if [ "$CONTINUATION_TOKEN" != "" ]; then
|
||||
add_parameter "canonical_request" "continuation-token=$continuation_token"
|
||||
fi
|
||||
@@ -56,6 +61,9 @@ create_canonical_hash_sts_and_signature
|
||||
|
||||
curl_command+=(curl -ks -w "\"%{http_code}\"")
|
||||
url="'$AWS_ENDPOINT_URL/$bucket_name"
|
||||
if [ "$MARKER" != "" ]; then
|
||||
add_parameter "url" "marker=$marker"
|
||||
fi
|
||||
if [ "$CONTINUATION_TOKEN" != "" ]; then
|
||||
add_parameter "url" "continuation-token=$continuation_token"
|
||||
fi
|
||||
|
||||
@@ -66,19 +66,21 @@ test_common_put_bucket_acl() {
|
||||
grantee_type="Group"
|
||||
grantee_id="http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
else
|
||||
grantee_type="ID"
|
||||
grantee_type="CanonicalUser"
|
||||
grantee_id="$USERNAME_ONE"
|
||||
fi
|
||||
run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "$grantee_type" "$grantee_id" "READ" "$AWS_ACCESS_KEY_ID"
|
||||
assert_success
|
||||
|
||||
log 5 "acl: $(cat "$TEST_FILE_FOLDER/$acl_file")"
|
||||
|
||||
run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file"
|
||||
assert_success
|
||||
|
||||
run get_check_acl_after_first_put "$1" "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
|
||||
run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$USERNAME_ONE" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"
|
||||
run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$USERNAME_ONE" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"
|
||||
assert_success
|
||||
|
||||
run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file"
|
||||
|
||||
@@ -36,6 +36,7 @@ source ./tests/util/util_list_buckets.sh
|
||||
source ./tests/util/util_list_objects.sh
|
||||
source ./tests/util/util_list_parts.sh
|
||||
source ./tests/util/util_lock_config.sh
|
||||
source ./tests/util/util_multipart_before_completion.sh
|
||||
source ./tests/util/util_ownership.sh
|
||||
source ./tests/util/util_policy.sh
|
||||
source ./tests/util/util_public_access_block.sh
|
||||
@@ -318,7 +319,7 @@ export RUN_USERS=true
|
||||
|
||||
@test "REST - get object attributes" {
|
||||
if [ "$DIRECT" != "true" ]; then
|
||||
skip "https://github.com/versity/versitygw/issues/916"
|
||||
skip "https://github.com/versity/versitygw/issues/1000"
|
||||
fi
|
||||
test_file="test_file"
|
||||
|
||||
@@ -340,7 +341,7 @@ export RUN_USERS=true
|
||||
|
||||
@test "REST - attributes - invalid param" {
|
||||
if [ "$DIRECT" != "true" ]; then
|
||||
skip "https://github.com/versity/versitygw/issues/917"
|
||||
skip "https://github.com/versity/versitygw/issues/1001"
|
||||
fi
|
||||
test_file="test_file"
|
||||
|
||||
@@ -359,7 +360,7 @@ export RUN_USERS=true
|
||||
|
||||
@test "REST - attributes - checksum" {
|
||||
if [ "$DIRECT" != "true" ]; then
|
||||
skip "https://github.com/versity/versitygw/issues/928"
|
||||
skip "https://github.com/versity/versitygw/issues/1006"
|
||||
fi
|
||||
test_file="test_file"
|
||||
|
||||
@@ -461,7 +462,7 @@ export RUN_USERS=true
|
||||
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_three" "$BUCKET_ONE_NAME" "$test_file_three"
|
||||
assert_success
|
||||
|
||||
run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two"
|
||||
run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" "TRUE"
|
||||
assert_success
|
||||
continuation_token=$output
|
||||
|
||||
@@ -469,3 +470,25 @@ export RUN_USERS=true
|
||||
run list_objects_check_continuation_error "$BUCKET_ONE_NAME" "${continuation_token:0:${#continuation_token}-3}"
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "REST - list objects v1 - no NextMarker without delimiter" {
|
||||
if [ "$DIRECT" != "true" ]; then
|
||||
skip "https://github.com/versity/versitygw/issues/999"
|
||||
fi
|
||||
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
|
||||
test_file="test_file"
|
||||
test_file_two="test_file_2"
|
||||
run create_test_files "$test_file" "$test_file_two"
|
||||
assert_success
|
||||
|
||||
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
|
||||
assert_success
|
||||
|
||||
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_two" "$BUCKET_ONE_NAME" "$test_file_two"
|
||||
assert_success
|
||||
|
||||
run list_objects_v1_check_nextmarker_empty "$BUCKET_ONE_NAME"
|
||||
assert_success
|
||||
}
|
||||
@@ -19,6 +19,7 @@ source ./tests/test_s3api_root_inner.sh
|
||||
source ./tests/util/util_file.sh
|
||||
source ./tests/util/util_multipart.sh
|
||||
source ./tests/util/util_multipart_abort.sh
|
||||
source ./tests/util/util_multipart_before_completion.sh
|
||||
source ./tests/util/util_tags.sh
|
||||
source ./tests/commands/get_object.sh
|
||||
source ./tests/commands/put_object.sh
|
||||
|
||||
@@ -21,6 +21,7 @@ source ./tests/test_s3api_policy_multipart.sh
|
||||
source ./tests/test_s3api_policy_object.sh
|
||||
source ./tests/util/util_multipart.sh
|
||||
source ./tests/util/util_multipart_abort.sh
|
||||
source ./tests/util/util_multipart_before_completion.sh
|
||||
source ./tests/util/util_file.sh
|
||||
source ./tests/util/util_policy.sh
|
||||
source ./tests/util/util_tags.sh
|
||||
|
||||
@@ -25,6 +25,7 @@ check_attributes_after_upload() {
|
||||
log 2 "'check_attributes_after_upload' requires file size"
|
||||
return 1
|
||||
fi
|
||||
log 5 "attributes: $(cat "$TEST_FILE_FOLDER/attributes.txt")"
|
||||
if ! object_size=$(xmllint --xpath '//*[local-name()="ObjectSize"]/text()' "$TEST_FILE_FOLDER/attributes.txt" 2>&1); then
|
||||
log 2 "error getting checksum: $object_size"
|
||||
return 1
|
||||
@@ -54,10 +55,11 @@ check_attributes_after_upload() {
|
||||
log 2 "unexpected parts count, expected 4, was $parts_count"
|
||||
return 1
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
check_attributes_invalid_param() {
|
||||
if [ "$1" -ne 1 ]; then
|
||||
if [ $# -ne 1 ]; then
|
||||
log 2 "'check_attributes_invalid_param' requires test file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -246,11 +246,11 @@ list_objects_with_user_rest_verify_success() {
|
||||
}
|
||||
|
||||
list_objects_check_params_get_token() {
|
||||
if [ $# -ne 3 ]; then
|
||||
log 2 "'list_objects_check_params_get_token' requires bucket name, files"
|
||||
if [ $# -ne 4 ]; then
|
||||
log 2 "'list_objects_check_params_get_token' requires bucket name, files, version two"
|
||||
return 1
|
||||
fi
|
||||
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
|
||||
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="$4" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
|
||||
log 2 "error attempting to get bucket ACL response: $result"
|
||||
return 1
|
||||
fi
|
||||
@@ -300,4 +300,31 @@ list_objects_check_continuation_error() {
|
||||
log 2 "invalid error code"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
}
|
||||
|
||||
list_objects_v1_check_nextmarker_empty() {
|
||||
if [ $# -ne 1 ]; then
|
||||
log 2 "'get_next_objects_v1' requires bucket name"
|
||||
return 1
|
||||
fi
|
||||
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
|
||||
log 2 "error attempting to get bucket ACL response: $result"
|
||||
return 1
|
||||
fi
|
||||
log 5 "output: $(cat "$TEST_FILE_FOLDER/objects.txt")"
|
||||
if ! next_marker=$(xmllint --xpath '//*[local-name()="NextMarker"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then
|
||||
if [[ "$next_marker" != *"XPath set is empty"* ]]; then
|
||||
log 2 "unexpected error: $next_marker"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
log 5 "next marker: $next_marker"
|
||||
marker_text=$(xmllint --xpath 'string(/NextMarker)' <(echo "$next_marker") 2>&1)
|
||||
log 5 "marker text: $marker_text"
|
||||
if [[ "$marker_text" != *"Document is empty"* ]]; then
|
||||
log 2 "NextMarker text should be empty, but is $marker_text"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -14,208 +14,6 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
create_upload_and_test_parts_listing() {
|
||||
if [ $# -ne 2 ]; then
|
||||
log 2 "'create_upload_and_test_parts_listing' requires test file, policy_file"
|
||||
return 1
|
||||
fi
|
||||
if ! create_multipart_upload_with_user "$BUCKET_ONE_NAME" "$1" "$USERNAME_ONE" "$PASSWORD_ONE"; then
|
||||
log 2 "error creating multipart upload with user"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2154
|
||||
if list_parts_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$1" "$upload_id"; then
|
||||
log 2 "list parts with user succeeded despite lack of policy permissions"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! setup_policy_with_single_statement "$TEST_FILE_FOLDER/$2" "2012-10-17" "Allow" "$USERNAME_ONE" "s3:ListMultipartUploadParts" "arn:aws:s3:::$BUCKET_ONE_NAME/*"; then
|
||||
log 2 "error setting up policy"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$2"; then
|
||||
log 2 "error putting policy"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! list_parts_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$1" "$upload_id"; then
|
||||
log 2 "error listing parts after policy add"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
start_multipart_upload_list_check_parts() {
|
||||
if [ $# -ne 3 ]; then
|
||||
log 2 "'start_multipart_upload_and_list_parts' requires bucket, key, original source"
|
||||
return 1
|
||||
fi
|
||||
if ! start_multipart_upload_and_list_parts "$1" "$2" "$3" 4; then
|
||||
log 2 "error starting upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
declare -a parts_map
|
||||
# shellcheck disable=SC2154
|
||||
log 5 "parts: $parts"
|
||||
for i in {0..3}; do
|
||||
if ! parse_parts_and_etags "$i"; then
|
||||
log 2 "error parsing part $i"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
if [[ ${#parts_map[@]} -eq 0 ]]; then
|
||||
log 2 "error loading multipart upload parts to check"
|
||||
return 1
|
||||
fi
|
||||
|
||||
for i in {0..3}; do
|
||||
if ! compare_parts_to_listed_parts "$i"; then
|
||||
log 2 "error comparing parts to listed parts"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
parse_parts_and_etags() {
|
||||
if [ $# -ne 1 ]; then
|
||||
log 2 "'parse_parts_and_etags' requires part id"
|
||||
return 1
|
||||
fi
|
||||
local part_number
|
||||
local etag
|
||||
# shellcheck disable=SC2154
|
||||
if ! part=$(echo "$parts" | grep -v "InsecureRequestWarning" | jq -r ".[$i]" 2>&1); then
|
||||
log 2 "error getting part: $part"
|
||||
return 1
|
||||
fi
|
||||
if ! part_number=$(echo "$part" | jq ".PartNumber" 2>&1); then
|
||||
log 2 "error parsing part number: $part_number"
|
||||
return 1
|
||||
fi
|
||||
if [[ $part_number == "" ]]; then
|
||||
log 2 "error: blank part number"
|
||||
return 1
|
||||
fi
|
||||
if ! etag=$(echo "$part" | jq ".ETag" 2>&1); then
|
||||
log 2 "error parsing etag: $etag"
|
||||
return 1
|
||||
fi
|
||||
if [[ $etag == "" ]]; then
|
||||
log 2 "error: blank etag"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2004
|
||||
parts_map[$part_number]=$etag
|
||||
}
|
||||
|
||||
compare_parts_to_listed_parts() {
|
||||
if [ $# -ne 1 ]; then
|
||||
log 2 "'compare_parts_to_listed_parts' requires part number"
|
||||
return 1
|
||||
fi
|
||||
local part_number
|
||||
local etag
|
||||
# shellcheck disable=SC2154
|
||||
if ! listed_part=$(echo "$listed_parts" | grep -v "InsecureRequestWarning" | jq -r ".Parts[$i]" 2>&1); then
|
||||
log 2 "error parsing listed part: $listed_part"
|
||||
return 1
|
||||
fi
|
||||
if ! part_number=$(echo "$listed_part" | jq ".PartNumber" 2>&1); then
|
||||
log 2 "error parsing listed part number: $part_number"
|
||||
return 1
|
||||
fi
|
||||
if ! etag=$(echo "$listed_part" | jq ".ETag" 2>&1); then
|
||||
log 2 "error getting listed etag: $etag"
|
||||
return 1
|
||||
fi
|
||||
if [[ ${parts_map[$part_number]} != "$etag" ]]; then
|
||||
log 2 "error: etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# list parts of an unfinished multipart upload
|
||||
# params: bucket, key, local file location, and parts to split into before upload
|
||||
# export parts on success, return 1 for error
|
||||
start_multipart_upload_and_list_parts() {
|
||||
if [ $# -ne 4 ]; then
|
||||
log 2 "list multipart upload parts command requires bucket, key, file, and part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! list_parts "$1" "$2" "$upload_id"; then
|
||||
log 2 "Error listing multipart upload parts: $listed_parts"
|
||||
return 1
|
||||
fi
|
||||
export listed_parts
|
||||
}
|
||||
|
||||
create_list_check_multipart_uploads() {
|
||||
if [ $# -ne 3 ]; then
|
||||
log 2 "list multipart uploads command requires bucket and two keys"
|
||||
return 1
|
||||
fi
|
||||
if ! create_and_list_multipart_uploads "$1" "$2" "$3"; then
|
||||
log 2 "error creating and listing multipart uploads"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2154
|
||||
log 5 "Uploads: $uploads"
|
||||
raw_uploads=$(echo "$uploads" | grep -v "InsecureRequestWarning")
|
||||
if ! key_one=$(echo "$raw_uploads" | jq -r '.Uploads[0].Key' 2>&1); then
|
||||
log 2 "error getting key one: $key_one"
|
||||
return 1
|
||||
fi
|
||||
if ! key_two=$(echo "$raw_uploads" | jq -r '.Uploads[1].Key' 2>&1); then
|
||||
log 2 "error getting key two: $key_two"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$2" != "$key_one" ]]; then
|
||||
log 2 "Key mismatch ($2, $key_one)"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$3" != "$key_two" ]]; then
|
||||
log 2 "Key mismatch ($3, $key_two)"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# list unfinished multipart uploads
|
||||
# params: bucket, key one, key two
|
||||
# export current two uploads on success, return 1 for error
|
||||
create_and_list_multipart_uploads() {
|
||||
if [ $# -ne 3 ]; then
|
||||
log 2 "list multipart uploads command requires bucket and two keys"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! create_multipart_upload "$1" "$2"; then
|
||||
log 2 "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! create_multipart_upload "$1" "$3"; then
|
||||
log 2 "error creating multpart upload two"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! list_multipart_uploads "$1"; then
|
||||
log 2 "error listing uploads"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload_from_bucket() {
|
||||
if [ $# -ne 4 ]; then
|
||||
log 2 "multipart upload from bucket command missing bucket, copy source, key, and/or part count"
|
||||
@@ -242,10 +40,12 @@ multipart_upload_from_bucket() {
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
# shellcheck disable=SC2154
|
||||
if ! upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i"; then
|
||||
log 2 "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2154
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
@@ -301,114 +101,6 @@ multipart_upload_from_bucket_range() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# perform all parts of a multipart upload before completion command
|
||||
# params: bucket, key, file to split and upload, number of file parts to upload
|
||||
# return: 0 for success, 1 for failure
|
||||
multipart_upload_before_completion() {
|
||||
if [ $# -ne 4 ]; then
|
||||
log 2 "multipart upload pre-completion command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! split_file "$3" "$4"; then
|
||||
log 2 "error splitting file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! create_multipart_upload "$1" "$2"; then
|
||||
log 2 "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
# shellcheck disable=SC2154
|
||||
if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then
|
||||
log 2 "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
export parts
|
||||
}
|
||||
|
||||
multipart_upload_before_completion_with_params() {
|
||||
if [ $# -ne 10 ]; then
|
||||
log 2 "multipart upload command missing bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging"
|
||||
return 1
|
||||
fi
|
||||
|
||||
split_file "$3" "$4" || split_result=$?
|
||||
if [[ $split_result -ne 0 ]]; then
|
||||
log 2 "error splitting file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
create_multipart_upload_params "$1" "$2" "$5" "$6" "$7" "$8" "$9" "${10}" || local create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
log 2 "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
upload_part "$1" "$2" "$upload_id" "$3" "$i" || local upload_result=$?
|
||||
if [[ $upload_result -ne 0 ]]; then
|
||||
log 2 "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
export parts
|
||||
}
|
||||
|
||||
multipart_upload_before_completion_custom() {
|
||||
if [ $# -lt 4 ]; then
|
||||
log 2 "multipart upload custom command missing bucket, key, file, part count, and/or optional params"
|
||||
return 1
|
||||
fi
|
||||
|
||||
split_file "$3" "$4" || local split_result=$?
|
||||
if [[ $split_result -ne 0 ]]; then
|
||||
log 2 "error splitting file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086 disable=SC2048
|
||||
create_multipart_upload_custom "$1" "$2" ${*:5} || local create_result=$?
|
||||
if [[ $create_result -ne 0 ]]; then
|
||||
log 2 "error creating multipart upload"
|
||||
return 1
|
||||
fi
|
||||
log 5 "upload ID: $upload_id"
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
upload_part "$1" "$2" "$upload_id" "$3" "$i" || local upload_result=$?
|
||||
if [[ $upload_result -ne 0 ]]; then
|
||||
log 2 "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
export parts
|
||||
}
|
||||
|
||||
multipart_upload_custom() {
|
||||
if [ $# -lt 4 ]; then
|
||||
log 2 "multipart upload custom command missing bucket, key, file, part count, and/or optional additional params"
|
||||
@@ -416,15 +108,13 @@ multipart_upload_custom() {
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086 disable=SC2048
|
||||
multipart_upload_before_completion_custom "$1" "$2" "$3" "$4" ${*:5} || local result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
if ! multipart_upload_before_completion_custom "$1" "$2" "$3" "$4" ${*:5}; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log 5 "upload ID: $upload_id, parts: $parts"
|
||||
complete_multipart_upload "$1" "$2" "$upload_id" "$parts" || local completed=$?
|
||||
if [[ $completed -ne 0 ]]; then
|
||||
if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then
|
||||
log 2 "Error completing upload"
|
||||
return 1
|
||||
fi
|
||||
@@ -437,14 +127,12 @@ multipart_upload() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
multipart_upload_before_completion "$1" "$2" "$3" "$4" || local result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
complete_multipart_upload "$1" "$2" "$upload_id" "$parts" || local completed=$?
|
||||
if [[ $completed -ne 0 ]]; then
|
||||
if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then
|
||||
log 2 "Error completing upload"
|
||||
return 1
|
||||
fi
|
||||
@@ -461,101 +149,20 @@ multipart_upload_with_params() {
|
||||
fi
|
||||
log 5 "1: $1, 2: $2, 3: $3, 4: $4, 5: $5, 6: $6, 7: $7, 8: $8, 9: $9, 10: ${10}"
|
||||
|
||||
multipart_upload_before_completion_with_params "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" || result=$?
|
||||
if [[ $result -ne 0 ]]; then
|
||||
if ! multipart_upload_before_completion_with_params "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}"; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
log 5 "Upload parts: $parts"
|
||||
|
||||
complete_multipart_upload "$1" "$2" "$upload_id" "$parts" || local completed=$?
|
||||
if [[ $completed -ne 0 ]]; then
|
||||
if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then
|
||||
log 2 "Error completing upload"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
create_upload_and_get_id_rest() {
|
||||
if [ $# -ne 2 ]; then
|
||||
log 2 "'create_upload_and_get_id_rest' requires bucket, key"
|
||||
return 1
|
||||
fi
|
||||
if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY=$2 OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" ./tests/rest_scripts/create_multipart_upload.sh); then
|
||||
log 2 "error creating multipart upload: $result"
|
||||
return 1
|
||||
fi
|
||||
if [ "$result" != "200" ]; then
|
||||
log 2 "error: response code: $result, output: $(cat "$TEST_FILE_FOLDER/output.txt")"
|
||||
return 1
|
||||
fi
|
||||
log 5 "multipart upload create info: $(cat "$TEST_FILE_FOLDER/output.txt")"
|
||||
if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/output.txt" 2>&1); then
|
||||
log 2 "error getting upload ID: $upload_id"
|
||||
return 1
|
||||
fi
|
||||
log 5 "upload ID: $upload_id"
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload_range_too_large() {
|
||||
if [ $# -ne 3 ]; then
|
||||
log 2 "'multipart_upload_range_too_large' requires bucket name, key, file location"
|
||||
return 1
|
||||
fi
|
||||
if multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-1000000000"; then
|
||||
log 2 "multipart upload succeeded despite overly large range"
|
||||
return 1
|
||||
fi
|
||||
log 5 "error: $upload_part_copy_error"
|
||||
if [[ $upload_part_copy_error != *"Range specified is not valid"* ]] && [[ $upload_part_copy_error != *"InvalidRange"* ]]; then
|
||||
log 2 "unexpected error: $upload_part_copy_error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
list_and_check_upload() {
|
||||
if [ $# -lt 2 ]; then
|
||||
log 2 "'list_and_check_upload' requires bucket, key, upload ID (optional)"
|
||||
return 1
|
||||
fi
|
||||
if ! uploads=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/uploads.txt" ./tests/rest_scripts/list_multipart_uploads.sh); then
|
||||
log 2 "error listing multipart uploads before upload: $result"
|
||||
return 1
|
||||
fi
|
||||
if ! upload_count=$(xmllint --xpath 'count(//*[local-name()="Upload"])' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then
|
||||
log 2 "error retrieving upload count: $upload_count"
|
||||
return 1
|
||||
fi
|
||||
if [[ (( $# == 2 ) && ( $upload_count != 0 )) ]]; then
|
||||
log 2 "upload count mismatch (expected 0, actual $upload_count)"
|
||||
return 1
|
||||
elif [[ (( $# == 3 ) && ( $upload_count != 1 )) ]]; then
|
||||
log 2 "upload count mismatch (expected 1, actual $upload_count)"
|
||||
return 1
|
||||
fi
|
||||
if [ $# -eq 2 ]; then
|
||||
return 0
|
||||
fi
|
||||
if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then
|
||||
log 2 "error retrieving key: $key"
|
||||
return 1
|
||||
fi
|
||||
if [ "$key" != "$2" ]; then
|
||||
log 2 "key mismatch (expected '$2', actual '$key')"
|
||||
return 1
|
||||
fi
|
||||
if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then
|
||||
log 2 "error retrieving upload ID: $upload_id"
|
||||
return 1
|
||||
fi
|
||||
if [ "$upload_id" != "$3" ]; then
|
||||
log 2 "upload ID mismatch (expected '$3', actual '$upload_id')"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
run_and_verify_multipart_upload_with_valid_range() {
|
||||
if [ $# -ne 3 ]; then
|
||||
@@ -582,25 +189,3 @@ run_and_verify_multipart_upload_with_valid_range() {
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
list_check_multipart_upload_key() {
|
||||
if [ $# -ne 4 ]; then
|
||||
log 2 "'list_check_multipart_upload_key' requires bucket, username, password, expected key"
|
||||
return 1
|
||||
fi
|
||||
if ! list_multipart_uploads_with_user "$1" "$2" "$3"; then
|
||||
log 2 "error listing multipart uploads with user"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2154
|
||||
log 5 "$uploads"
|
||||
if ! upload_key=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -r ".Uploads[0].Key" 2>&1); then
|
||||
log 2 "error parsing upload key from uploads message: $upload_key"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$4" != "$upload_key" ]]; then
|
||||
log 2 "upload key doesn't match file marked as being uploaded (expected: '$4', actual: '$upload_key')"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
423
tests/util/util_multipart_before_completion.sh
Normal file
423
tests/util/util_multipart_before_completion.sh
Normal file
@@ -0,0 +1,423 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2024 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
create_upload_and_test_parts_listing() {
|
||||
if [ $# -ne 2 ]; then
|
||||
log 2 "'create_upload_and_test_parts_listing' requires test file, policy_file"
|
||||
return 1
|
||||
fi
|
||||
if ! create_multipart_upload_with_user "$BUCKET_ONE_NAME" "$1" "$USERNAME_ONE" "$PASSWORD_ONE"; then
|
||||
log 2 "error creating multipart upload with user"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2154
|
||||
if list_parts_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$1" "$upload_id"; then
|
||||
log 2 "list parts with user succeeded despite lack of policy permissions"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! setup_policy_with_single_statement "$TEST_FILE_FOLDER/$2" "2012-10-17" "Allow" "$USERNAME_ONE" "s3:ListMultipartUploadParts" "arn:aws:s3:::$BUCKET_ONE_NAME/*"; then
|
||||
log 2 "error setting up policy"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$2"; then
|
||||
log 2 "error putting policy"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! list_parts_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$1" "$upload_id"; then
|
||||
log 2 "error listing parts after policy add"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
start_multipart_upload_list_check_parts() {
|
||||
if [ $# -ne 3 ]; then
|
||||
log 2 "'start_multipart_upload_and_list_parts' requires bucket, key, original source"
|
||||
return 1
|
||||
fi
|
||||
if ! start_multipart_upload_and_list_parts "$1" "$2" "$3" 4; then
|
||||
log 2 "error starting upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
declare -a parts_map
|
||||
# shellcheck disable=SC2154
|
||||
log 5 "parts: $parts"
|
||||
for i in {0..3}; do
|
||||
if ! parse_parts_and_etags "$i"; then
|
||||
log 2 "error parsing part $i"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
if [[ ${#parts_map[@]} -eq 0 ]]; then
|
||||
log 2 "error loading multipart upload parts to check"
|
||||
return 1
|
||||
fi
|
||||
|
||||
for i in {0..3}; do
|
||||
if ! compare_parts_to_listed_parts "$i"; then
|
||||
log 2 "error comparing parts to listed parts"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
parse_parts_and_etags() {
|
||||
if [ $# -ne 1 ]; then
|
||||
log 2 "'parse_parts_and_etags' requires part id"
|
||||
return 1
|
||||
fi
|
||||
local part_number
|
||||
local etag
|
||||
# shellcheck disable=SC2154
|
||||
if ! part=$(echo "$parts" | grep -v "InsecureRequestWarning" | jq -r ".[$i]" 2>&1); then
|
||||
log 2 "error getting part: $part"
|
||||
return 1
|
||||
fi
|
||||
if ! part_number=$(echo "$part" | jq ".PartNumber" 2>&1); then
|
||||
log 2 "error parsing part number: $part_number"
|
||||
return 1
|
||||
fi
|
||||
if [[ $part_number == "" ]]; then
|
||||
log 2 "error: blank part number"
|
||||
return 1
|
||||
fi
|
||||
if ! etag=$(echo "$part" | jq ".ETag" 2>&1); then
|
||||
log 2 "error parsing etag: $etag"
|
||||
return 1
|
||||
fi
|
||||
if [[ $etag == "" ]]; then
|
||||
log 2 "error: blank etag"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2004
|
||||
parts_map[$part_number]=$etag
|
||||
}
|
||||
|
||||
compare_parts_to_listed_parts() {
|
||||
if [ $# -ne 1 ]; then
|
||||
log 2 "'compare_parts_to_listed_parts' requires part number"
|
||||
return 1
|
||||
fi
|
||||
local part_number
|
||||
local etag
|
||||
# shellcheck disable=SC2154
|
||||
if ! listed_part=$(echo "$listed_parts" | grep -v "InsecureRequestWarning" | jq -r ".Parts[$i]" 2>&1); then
|
||||
log 2 "error parsing listed part: $listed_part"
|
||||
return 1
|
||||
fi
|
||||
if ! part_number=$(echo "$listed_part" | jq ".PartNumber" 2>&1); then
|
||||
log 2 "error parsing listed part number: $part_number"
|
||||
return 1
|
||||
fi
|
||||
if ! etag=$(echo "$listed_part" | jq ".ETag" 2>&1); then
|
||||
log 2 "error getting listed etag: $etag"
|
||||
return 1
|
||||
fi
|
||||
if [[ ${parts_map[$part_number]} != "$etag" ]]; then
|
||||
log 2 "error: etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# list parts of an unfinished multipart upload
|
||||
# params: bucket, key, local file location, and parts to split into before upload
|
||||
# export parts on success, return 1 for error
|
||||
start_multipart_upload_and_list_parts() {
|
||||
if [ $# -ne 4 ]; then
|
||||
log 2 "list multipart upload parts command requires bucket, key, file, and part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then
|
||||
log 2 "error performing pre-completion multipart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! list_parts "$1" "$2" "$upload_id"; then
|
||||
log 2 "Error listing multipart upload parts: $listed_parts"
|
||||
return 1
|
||||
fi
|
||||
export listed_parts
|
||||
}
|
||||
|
||||
create_list_check_multipart_uploads() {
|
||||
if [ $# -ne 3 ]; then
|
||||
log 2 "list multipart uploads command requires bucket and two keys"
|
||||
return 1
|
||||
fi
|
||||
if ! create_and_list_multipart_uploads "$1" "$2" "$3"; then
|
||||
log 2 "error creating and listing multipart uploads"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2154
|
||||
log 5 "Uploads: $uploads"
|
||||
raw_uploads=$(echo "$uploads" | grep -v "InsecureRequestWarning")
|
||||
if ! key_one=$(echo "$raw_uploads" | jq -r '.Uploads[0].Key' 2>&1); then
|
||||
log 2 "error getting key one: $key_one"
|
||||
return 1
|
||||
fi
|
||||
if ! key_two=$(echo "$raw_uploads" | jq -r '.Uploads[1].Key' 2>&1); then
|
||||
log 2 "error getting key two: $key_two"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$2" != "$key_one" ]]; then
|
||||
log 2 "Key mismatch ($2, $key_one)"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$3" != "$key_two" ]]; then
|
||||
log 2 "Key mismatch ($3, $key_two)"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# list unfinished multipart uploads
|
||||
# params: bucket, key one, key two
|
||||
# export current two uploads on success, return 1 for error
|
||||
create_and_list_multipart_uploads() {
|
||||
if [ $# -ne 3 ]; then
|
||||
log 2 "list multipart uploads command requires bucket and two keys"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! create_multipart_upload "$1" "$2"; then
|
||||
log 2 "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! create_multipart_upload "$1" "$3"; then
|
||||
log 2 "error creating multpart upload two"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! list_multipart_uploads "$1"; then
|
||||
log 2 "error listing uploads"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# perform all parts of a multipart upload before completion command
|
||||
# params: bucket, key, file to split and upload, number of file parts to upload
|
||||
# return: 0 for success, 1 for failure
|
||||
multipart_upload_before_completion() {
|
||||
if [ $# -ne 4 ]; then
|
||||
log 2 "multipart upload pre-completion command missing bucket, key, file, and/or part count"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! split_file "$3" "$4"; then
|
||||
log 2 "error splitting file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! create_multipart_upload "$1" "$2"; then
|
||||
log 2 "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
# shellcheck disable=SC2154
|
||||
if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then
|
||||
log 2 "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
export parts
|
||||
}
|
||||
|
||||
multipart_upload_before_completion_with_params() {
|
||||
if [ $# -ne 10 ]; then
|
||||
log 2 "multipart upload command missing bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! result=$(split_file "$3" "$4" 2>&1); then
|
||||
log 2 "error splitting file: $result"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! create_multipart_upload_params "$1" "$2" "$5" "$6" "$7" "$8" "$9" "${10}"; then
|
||||
log 2 "error creating multpart upload"
|
||||
return 1
|
||||
fi
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then
|
||||
log 2 "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
export parts
|
||||
}
|
||||
|
||||
multipart_upload_before_completion_custom() {
|
||||
if [ $# -lt 4 ]; then
|
||||
log 2 "multipart upload custom command missing bucket, key, file, part count, and/or optional params"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! result=$(split_file "$3" "$4" 2>&1); then
|
||||
log 2 "error splitting file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086 disable=SC2048
|
||||
if ! create_multipart_upload_custom "$1" "$2" ${*:5}; then
|
||||
log 2 "error creating multipart upload"
|
||||
return 1
|
||||
fi
|
||||
log 5 "upload ID: $upload_id"
|
||||
|
||||
parts="["
|
||||
for ((i = 1; i <= $4; i++)); do
|
||||
if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then
|
||||
log 2 "error uploading part $i"
|
||||
return 1
|
||||
fi
|
||||
parts+="{\"ETag\": $etag, \"PartNumber\": $i}"
|
||||
if [[ $i -ne $4 ]]; then
|
||||
parts+=","
|
||||
fi
|
||||
done
|
||||
parts+="]"
|
||||
|
||||
export parts
|
||||
}
|
||||
|
||||
create_upload_and_get_id_rest() {
|
||||
if [ $# -ne 2 ]; then
|
||||
log 2 "'create_upload_and_get_id_rest' requires bucket, key"
|
||||
return 1
|
||||
fi
|
||||
if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY=$2 OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" ./tests/rest_scripts/create_multipart_upload.sh); then
|
||||
log 2 "error creating multipart upload: $result"
|
||||
return 1
|
||||
fi
|
||||
if [ "$result" != "200" ]; then
|
||||
log 2 "error: response code: $result, output: $(cat "$TEST_FILE_FOLDER/output.txt")"
|
||||
return 1
|
||||
fi
|
||||
log 5 "multipart upload create info: $(cat "$TEST_FILE_FOLDER/output.txt")"
|
||||
if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/output.txt" 2>&1); then
|
||||
log 2 "error getting upload ID: $upload_id"
|
||||
return 1
|
||||
fi
|
||||
log 5 "upload ID: $upload_id"
|
||||
return 0
|
||||
}
|
||||
|
||||
multipart_upload_range_too_large() {
|
||||
if [ $# -ne 3 ]; then
|
||||
log 2 "'multipart_upload_range_too_large' requires bucket name, key, file location"
|
||||
return 1
|
||||
fi
|
||||
if multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-1000000000"; then
|
||||
log 2 "multipart upload succeeded despite overly large range"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2154
|
||||
log 5 "error: $upload_part_copy_error"
|
||||
if [[ $upload_part_copy_error != *"Range specified is not valid"* ]] && [[ $upload_part_copy_error != *"InvalidRange"* ]]; then
|
||||
log 2 "unexpected error: $upload_part_copy_error"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
list_and_check_upload() {
|
||||
if [ $# -lt 2 ]; then
|
||||
log 2 "'list_and_check_upload' requires bucket, key, upload ID (optional)"
|
||||
return 1
|
||||
fi
|
||||
if ! uploads=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/uploads.txt" ./tests/rest_scripts/list_multipart_uploads.sh); then
|
||||
log 2 "error listing multipart uploads before upload: $result"
|
||||
return 1
|
||||
fi
|
||||
if ! upload_count=$(xmllint --xpath 'count(//*[local-name()="Upload"])' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then
|
||||
log 2 "error retrieving upload count: $upload_count"
|
||||
return 1
|
||||
fi
|
||||
if [[ (( $# == 2 ) && ( $upload_count != 0 )) ]]; then
|
||||
log 2 "upload count mismatch (expected 0, actual $upload_count)"
|
||||
return 1
|
||||
elif [[ (( $# == 3 ) && ( $upload_count != 1 )) ]]; then
|
||||
log 2 "upload count mismatch (expected 1, actual $upload_count)"
|
||||
return 1
|
||||
fi
|
||||
if [ $# -eq 2 ]; then
|
||||
return 0
|
||||
fi
|
||||
if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then
|
||||
log 2 "error retrieving key: $key"
|
||||
return 1
|
||||
fi
|
||||
if [ "$key" != "$2" ]; then
|
||||
log 2 "key mismatch (expected '$2', actual '$key')"
|
||||
return 1
|
||||
fi
|
||||
if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then
|
||||
log 2 "error retrieving upload ID: $upload_id"
|
||||
return 1
|
||||
fi
|
||||
if [ "$upload_id" != "$3" ]; then
|
||||
log 2 "upload ID mismatch (expected '$3', actual '$upload_id')"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
list_check_multipart_upload_key() {
|
||||
if [ $# -ne 4 ]; then
|
||||
log 2 "'list_check_multipart_upload_key' requires bucket, username, password, expected key"
|
||||
return 1
|
||||
fi
|
||||
if ! list_multipart_uploads_with_user "$1" "$2" "$3"; then
|
||||
log 2 "error listing multipart uploads with user"
|
||||
return 1
|
||||
fi
|
||||
# shellcheck disable=SC2154
|
||||
log 5 "$uploads"
|
||||
if ! upload_key=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -r ".Uploads[0].Key" 2>&1); then
|
||||
log 2 "error parsing upload key from uploads message: $upload_key"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$4" != "$upload_key" ]]; then
|
||||
log 2 "upload key doesn't match file marked as being uploaded (expected: '$4', actual: '$upload_key')"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
Reference in New Issue
Block a user