diff --git a/tests/Dockerfile_test_bats b/tests/Dockerfile_test_bats index b1257cc1..95e0c9bb 100644 --- a/tests/Dockerfile_test_bats +++ b/tests/Dockerfile_test_bats @@ -36,6 +36,8 @@ ENV PATH="$VENV/bin:${PATH}" WORKDIR /home/tester RUN make +RUN chown -R tester:tester /home/tester/.cache/go-build /home/tester/go + USER tester RUN . $SECRETS_FILE && \ @@ -55,5 +57,14 @@ RUN mkdir config log && cp $CONFIG_FILE config/.env ENV WORKSPACE=. ENV VERSITYGW_TEST_ENV=/home/tester/config/.env +#RUN set -x; \ +# id; \ +# whoami; \ +# echo "GOCACHE=$(go env GOCACHE)"; \ +# echo "GOMODCACHE=$(go env GOMODCACHE)"; \ +# echo "GOPATH=$(go env GOPATH)" || true +# +#RUN cache="$(go env GOCACHE)" && ls -ld "$cache" && find "$cache" -maxdepth 2 -ls 2>/dev/null | head -100 || true + ENTRYPOINT ["tests/run.sh"] CMD ["s3api,s3,s3cmd,mc,rest"] diff --git a/tests/commands/put_bucket_policy.sh b/tests/commands/put_bucket_policy.sh index e66602f8..0b22596e 100644 --- a/tests/commands/put_bucket_policy.sh +++ b/tests/commands/put_bucket_policy.sh @@ -90,37 +90,3 @@ put_bucket_policy_rest_200_or_204() { fi return 0 } - -create_website_with_random_string_and_add_permissions() { - if ! check_param_count_v2 "bucket name" 1 $#; then - return 1 - fi - local bucket_name="$1" - local response random_string policy_file - - if ! response=$(create_website_with_random_string "$bucket_name" 2>&1); then - log 2 "error creating website: $response" - return 1 - fi - read -r _ random_string <<< "$response" - - if [ "$DIRECT" == "true" ]; then - if ! put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false"; then - log 2 "error putting public access block" - return 1 - fi - fi - - if ! response=$(setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" 2>&1); then - log 2 "error setting up policy: $response" - return 1 - fi - policy_file="$response" - - if ! put_bucket_policy "rest" "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"; then - log 2 "error putting bucket policy" - return 1 - fi - printf '%s\n' "$random_string" - return 0 -} diff --git a/tests/commands/upload_part_copy.sh b/tests/commands/upload_part_copy.sh index 6cbcacf5..6f1e8765 100644 --- a/tests/commands/upload_part_copy.sh +++ b/tests/commands/upload_part_copy.sh @@ -15,34 +15,37 @@ # under the License. upload_part_copy() { - if [ $# -ne 5 ]; then - log 2 "upload multipart part copy function must have bucket, key, upload ID, file name, part number" + if ! check_param_count_v2 "bucket, key, upload ID, file name, part number" 5 $#; then return 1 fi - local etag_json - log 5 "parameters: '$1' '$2' '$3' '$4' '$5'" - etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))") || local uploaded=$? - if [[ $uploaded -ne 0 ]]; then - log 2 "Error uploading part $5: $etag_json" + local bucket="$1" key="$2" upload_id="$3" file_name="$4" part_number="$5" + local response response_without_warning + + if ! response=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" 2>&1); then + log 2 "UploadPartCopy command error: $response" return 1 fi - etag=$(echo "$etag_json" | jq '.CopyPartResult.ETag') - export etag + log 5 "UploadPartCopy response: $response" + response_without_warning=$(grep -v "InsecureRequestWarning" - <<< "$response") + etag=$(jq -r '.CopyPartResult.ETag' - <<< "$response_without_warning" 2>&1) + printf '%s\n' "$etag" + return 0 } upload_part_copy_with_range() { - if [ $# -ne 6 ]; then - log 2 "upload multipart part copy function must have bucket, key, upload ID, file name, part number, range" + if ! check_param_count_v2 "bucket, key, upload ID, file name, part number, range" 6 $#; then return 1 fi - local etag_json - log 5 "bucket: $1, key: $2, upload ID: $3, file name: $4, range: $5, copy source range: $6" - etag_json=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$1" --key "$2" --upload-id "$3" --part-number "$5" --copy-source "$1/$4-$(($5-1))" --copy-source-range "$6" 2>&1) || local uploaded=$? - if [[ $uploaded -ne 0 ]]; then - log 2 "Error uploading part $5: $etag_json" - export upload_part_copy_error=$etag_json + local bucket="$1" key="$2" upload_id="$3" file_name="$4" part_number="$5" range="$6" + local response etag + + if ! response=$(send_command aws --no-verify-ssl s3api upload-part-copy --bucket "$bucket" --key "$key" --upload-id "$upload_id" \ + --part-number "$part_number" --copy-source "${bucket}/${file_name}-$((part_number-1))" --copy-source-range "$range" 2>&1); then + printf "s3api UploadPartCopy command error: %s\n" "$response" return 1 fi - etag=$(echo "$etag_json" | grep -v "InsecureRequestWarning" | jq '.CopyPartResult.ETag') - export etag + log 5 "UploadPartCopy with range response: $response" + etag=$(echo "$response" | grep -v "InsecureRequestWarning" | jq -r '.CopyPartResult.ETag') + printf '%s\n' "$etag" + return 0 } \ No newline at end of file diff --git a/tests/download_dependencies.sh b/tests/download_dependencies.sh index c99dcba2..d87a8d7f 100644 --- a/tests/download_dependencies.sh +++ b/tests/download_dependencies.sh @@ -13,6 +13,15 @@ download_apt_packages() { return 1 fi + if ! env DEBIAN_FRONTEND=noninteractive sudo rm -rf /var/lib/apt/lists/*; then + echo "error removing old apt lists" >&2 + return 1 + fi + if ! env DEBIAN_FRONTEND=noninteractive sudo apt-get clean; then + echo "error with apt clean" >&2 + return 1 + fi + if ! env DEBIAN_FRONTEND=noninteractive sudo apt-get update; then echo "error with apt update" >&2 return 1 diff --git a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh index 7526db8e..985371b8 100644 --- a/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh +++ b/tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh @@ -18,22 +18,25 @@ calculate_multipart_checksum() { if ! check_param_count_gt "checksum type, part count, data file, lowercase algorithm, checksums" 4 $#; then return 1 fi - log 5 "checksums: ${*:5}" - if [ "$1" == "COMPOSITE" ]; then - if ! response=$(calculate_composite_checksum "$4" "${@:5}" 2>&1); then + local checksum_type="$1" part_count="$2" data_file="$3" lowercase_algorithm="$4" checksums=("${@:5}") + local response + + log 5 "checksums: ${checksums[*]}" + if [ "$checksum_type" == "COMPOSITE" ]; then + if ! response=$(calculate_composite_checksum "$lowercase_algorithm" "${checksums[@]}" 2>&1); then log 2 "error calculating checksum: $response" return 1 fi - checksum="$response-$2" + checksum="$response-${part_count}" echo "$checksum" return 0 fi - if [ "$1" != "FULL_OBJECT" ]; then - log 2 "unrecognized checksum type: $1" + if [ "$checksum_type" != "FULL_OBJECT" ]; then + log 2 "unrecognized checksum type: '$checksum_type'" return 1 fi - if ! response=$(DATA_FILE="$3" CHECKSUM_TYPE="$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then + if ! response=$(DATA_FILE="$data_file" CHECKSUM_TYPE="$lowercase_algorithm" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then log 2 "error calculating checksum: $response" return 1 fi @@ -42,17 +45,21 @@ calculate_multipart_checksum() { } complete_multipart_upload_with_checksum() { - if ! check_param_count_v2 "bucket, key, file, upload ID, part count, parts payload, checksum type, checksum algorithm" 8 $#; then + if ! check_param_count_v2 "bucket, key, file, upload ID, part count, parts payload, checksum type, checksum algorithm, checksum" 9 $#; then return 1 fi - log 5 "checksum algorithm: $8" + local bucket="$1" key="$2" file="$3" upload_id="$4" part_count="$5" parts_payload="$6" checksum_type="$7" checksum_algorithm="$8" checksum="$9" + local response response_file result + + log 5 "checksum algorithm: '$checksum_algorithm'" if ! response=$(get_file_name 2>&1); then log 2 "error getting file name: $response" return 1 fi response_file="$response" - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$4" PARTS="$6" CHECKSUM_TYPE="$7" CHECKSUM_ALGORITHM="$8" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/$response_file" ./tests/rest_scripts/complete_multipart_upload.sh); then + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" UPLOAD_ID="$upload_id" PARTS="$parts_payload" CHECKSUM_TYPE="$checksum_type" \ + CHECKSUM_ALGORITHM="$checksum_algorithm" CHECKSUM_HASH="$checksum" OUTPUT_FILE="$TEST_FILE_FOLDER/$response_file" ./tests/rest_scripts/complete_multipart_upload.sh); then log 2 "error completing multipart upload" return 1 fi @@ -87,7 +94,7 @@ calculate_composite_checksum() { elif [ "$1" == "crc32" ]; then composite="$(gzip -c -1 "$TEST_FILE_FOLDER/all_checksums.bin" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)" elif [ "$1" == "crc32c" ]; then - if ! composite=$(CHECKSUM_TYPE="$1" DATA_FILE="$TEST_FILE_FOLDER/all_checksums.bin" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then + if ! composite=$(CHECKSUM_TYPE="$1" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" DATA_FILE="$TEST_FILE_FOLDER/all_checksums.bin" ./tests/rest_scripts/calculate_checksum.sh 2>&1); then log 2 "error calculating crc32c checksum: $composite" return 1 fi @@ -102,6 +109,9 @@ test_multipart_upload_with_checksum() { if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then return 1 fi + local checksum_type="$1" checksum_algorithm="$2" + local file_and_bucket bucket_name mp_file_name response checksum + if ! file_and_bucket=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then log 2 "error setting up file and bucket" return 1 @@ -109,7 +119,7 @@ test_multipart_upload_with_checksum() { read -r bucket_name mp_file_name <<< "$file_and_bucket" log 5 "file name: $mp_file_name, file info: $(ls -l "$TEST_FILE_FOLDER/$mp_file_name")" - if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then + if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$checksum_type" "$checksum_algorithm" 2>&1); then log 2 "error performing multipart upload with checksum before completion: $response" return 1 fi @@ -120,11 +130,13 @@ test_multipart_upload_with_checksum() { local payload_parts="${response_lines[3]}" read -r -a checksums <<< "$checksum_string" - if ! calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}"; then - log 2 "error calculating multipart checksum" + if ! response=$(calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}" 2>&1); then + log 2 "error calculating multipart checksum: $response" return 1 fi - if ! complete_multipart_upload_with_checksum "$bucket_name" "$mp_file_name" "$TEST_FILE_FOLDER/$mp_file_name" "$upload_id" 2 "$payload_parts" "$1" "$2"; then + checksum="$response" + + if ! complete_multipart_upload_with_checksum "$bucket_name" "$mp_file_name" "$TEST_FILE_FOLDER/$mp_file_name" "$upload_id" 2 "$payload_parts" "$1" "$2" "$checksum"; then log 2 "error completing multipart upload with checksum" return 1 fi @@ -135,12 +147,16 @@ test_complete_multipart_upload_unneeded_algorithm_parameter() { if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then return 1 fi + local checksum_type="$1" algorithm="$2" + local file_and_bucket bucket_name mp_file_name + lcoal -a response_lines + if ! file_and_bucket=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then log 2 "error setting up file and bucket" return 1 fi read -r bucket_name mp_file_name <<< "$file_and_bucket" - if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then + if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$checksum_type" "$algorithm" 2>&1); then log 2 "error performing multipart upload with checksum before completion" return 1 fi @@ -159,29 +175,33 @@ test_complete_multipart_upload_incorrect_checksum() { if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then return 1 fi + local checksum_type="$1" algorithm="$2" + local response bucket_name mp_file_name checksum + local -a response_lines + if ! response=$(setup_bucket_and_large_file_v3 "$BUCKET_ONE_NAME" 2>&1); then log 2 "error setting up file and bucket" return 1 fi read -r bucket_name mp_file_name <<< "$response" - if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then + if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$checksum_type" "$algorithm" 2>&1); then log 2 "error performing multipart upload with checksum before completion" return 1 fi - mapfile -t response_lines<<< "$response" + mapfile -t response_lines <<< "$response" local lowercase_checksum_algorithm="${response_lines[0]}" local upload_id="${response_lines[1]}" local checksum_string="${response_lines[2]}" local payload_parts="${response_lines[3]}" read -r -a checksums <<< "$checksum_string" - if ! response=$(calculate_multipart_checksum "$1" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}" 2>&1); then + if ! response=$(calculate_multipart_checksum "$checksum_type" 2 "$TEST_FILE_FOLDER/$mp_file_name" "$lowercase_checksum_algorithm" "${checksums[@]}" 2>&1); then log 2 "error calculating multipart checksum" return 1 fi checksum="$response" - if ! complete_multipart_upload_rest_incorrect_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$payload_parts" "$1" "$2" "$checksum"; then + if ! complete_multipart_upload_rest_incorrect_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$payload_parts" "$checksum_type" "$algorithm" "$checksum"; then log 2 "error completing multipart upload with incorrect checksum" return 1 fi @@ -192,12 +212,16 @@ test_complete_multipart_upload_invalid_checksum() { if ! check_param_count_v2 "checksum type, algorithm" 2 $#; then return 1 fi + local checksum_type="$1" algorithm="$2" + local file_and_bucket bucket_name mp_file_name response + local -a response_lines + if ! file_and_bucket=$(setup_bucket_and_file_v3 "$BUCKET_ONE_NAME" 2>&1); then log 2 "error setting up file and bucket" return 1 fi read -r bucket_name mp_file_name <<< "$file_and_bucket" - if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$1" "$2" 2>&1); then + if ! response=$(perform_full_multipart_upload_with_checksum_before_completion "$bucket_name" "$mp_file_name" "$checksum_type" "$algorithm" 2>&1); then log 2 "error performing multipart upload with checksum before completion: $response" return 1 fi @@ -205,7 +229,7 @@ test_complete_multipart_upload_invalid_checksum() { local upload_id="${response_lines[1]}" local parts_payload="${response_lines[3]}" - if ! complete_multipart_upload_rest_invalid_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload" "$1" "$2" "wrong"; then + if ! complete_multipart_upload_rest_invalid_checksum "$bucket_name" "$mp_file_name" "$upload_id" "$parts_payload" "$checksum_type" "$algorithm" "wrong"; then log 2 "error completing multipart upload with invalid checksum" return 1 fi @@ -216,13 +240,15 @@ complete_multipart_upload_invalid_object_size_string() { if ! check_param_count_v2 "bucket, key, file" 3 $#; then return 1 fi + local bucket="$1" key="$2" file="$3" local response upload_id parts_payload - if ! response=$(multipart_upload_rest_before_completion "$1" "$2" "$3" 2 2>&1); then + + if ! response=$(multipart_upload_rest_before_completion "$bucket" "$key" "$file" 2 2>&1); then log 2 "error performing multipart upload before completion: $response" return 1 fi read -r upload_id parts_payload <<< "$response" - if ! complete_multipart_upload_rest_expect_error "$1" "$2" "$upload_id" "$parts_payload" "MULTIPART_OBJECT_SIZE=size" "400" "InvalidRequest" "Value for x-amz-mp-object-size header is invalid"; then + if ! complete_multipart_upload_rest_expect_error "$bucket" "$key" "$upload_id" "$parts_payload" "MULTIPART_OBJECT_SIZE=size" "400" "InvalidRequest" "Value for x-amz-mp-object-size header is invalid"; then log 2 "error completing multipart upload" return 1 fi @@ -233,7 +259,9 @@ perform_multipart_upload_rest() { if ! check_param_count_v2 "bucket, key, four parts" 6 $#; then return 1 fi - if ! perform_multipart_upload_rest_variable_parts "$@"; then + local bucket="$1" key="$2" four_parts=("${@:3}") + + if ! perform_multipart_upload_rest_variable_parts "$bucket" "$key" "${four_parts[@]}"; then return 1 fi return 0 @@ -243,20 +271,22 @@ upload_check_parts() { if ! check_param_count_v2 "bucket, key, list of 4 parts" 6 $#; then return 1 fi - if ! upload_id=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then + local bucket="$1" key="$2" four_parts=("${@:3}") + local upload_id parts_payload + + if ! upload_id=$(create_multipart_upload_rest "$bucket" "$key" "" "parse_upload_id" 2>&1); then log 2 "error creating upload: $upload_id" return 1 fi - if ! check_part_list_rest "$1" "$2" "$upload_id" 0; then + if ! check_part_list_rest "$bucket" "$key" "$upload_id" 0; then log 2 "error checking part list before part upload" return 1 fi - if ! parts_payload=$(upload_each_part_and_check "$1" "$2" "$upload_id" "${@:3}" 2>&1); then + if ! parts_payload=$(upload_each_part_and_check "$bucket" "$key" "$upload_id" "${four_parts[@]}" 2>&1); then log 2 "error uploading and checking parts: $parts_payload" return 1 fi - log 5 "PARTS PAYLOAD: $parts_payload" - if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then + if ! complete_multipart_upload_rest "$bucket" "$key" "$upload_id" "$parts_payload"; then log 2 "error completing multipart upload" return 1 fi @@ -267,17 +297,18 @@ perform_multipart_upload_rest_variable_parts() { if ! check_param_count_gt "bucket, key, at least two part locations" 4 $#; then return 1 fi - local response + local bucket="$1" key="$2" parts=("${@:3}") + local response upload_id part etag - if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then + if ! response=$(create_multipart_upload_rest "$bucket" "$key" "" "parse_upload_id" 2>&1); then log 2 "error creating multipart upload: $response" return 1 fi upload_id="$response" local parts_payload="" idx=1 - for part in "${@:3}"; do - if ! response=$(upload_part_rest "$1" "$2" "$upload_id" "$idx" "$part" 2>&1); then + for part in "${parts[@]}"; do + if ! response=$(upload_part_rest "$bucket" "$key" "$upload_id" "$idx" "$part" 2>&1); then log 2 "error uploading part $idx: $response" return 1 fi @@ -287,7 +318,7 @@ perform_multipart_upload_rest_variable_parts() { done log 5 "final payload: $parts_payload" - if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then + if ! complete_multipart_upload_rest "$bucket" "$key" "$upload_id" "$parts_payload"; then log 2 "error completing multipart upload" return 1 fi @@ -298,15 +329,120 @@ complete_multipart_upload_empty_upload_id() { if ! check_param_count_v2 "bucket, key, file" 3 $#; then return 1 fi + local bucket="$1" key="$2" file="$3" local response upload_id parts_payload - if ! response=$(multipart_upload_rest_before_completion "$1" "$2" "$3" 2 2>&1); then + + if ! response=$(multipart_upload_rest_before_completion "$bucket" "$key" "$file" 2 2>&1); then log 2 "error performing multipart upload before completion: $response" return 1 fi read -r upload_id parts_payload <<< "$response" - if ! complete_multipart_upload_rest_expect_error "$1" "$2" "" "$parts_payload" "" "404" "NoSuchUpload" "The specified upload does not exist"; then + if ! complete_multipart_upload_rest_expect_error "$bucket" "$key" "" "$parts_payload" "" "404" "NoSuchUpload" "The specified upload does not exist"; then log 2 "error completing multipart upload" return 1 fi return 0 } + +create_upload_part_copy_rest() { + if ! check_param_count "create_upload_part_copy_rest" "bucket, key, >20MB file" 3 $#; then + return 1 + fi + local bucket="$1" key="$2" large_file="$3" + local response upload_id file_name parts_payload i part_number result etag + + if ! split_and_put_file "$bucket" "$key" "$large_file" 4; then + log 2 "error splitting and putting file" + return 1 + fi + + if ! response=$(create_multipart_upload_rest "$bucket" "$key" "" "parse_upload_id" 2>&1); then + log 2 "error creating upload and getting ID: $response" + return 1 + fi + upload_id="$response" + + if ! response=$(get_file_name 2>&1); then + log 2 "error getting file name: $response" + return 1 + fi + file_name="$response" + + parts_payload="" + for ((i=0; i<=3; i++)); do + part_number=$((i+1)) + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" PART_NUMBER="$part_number" UPLOAD_ID="$upload_id" PART_LOCATION="${bucket}/${key}-$i" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/upload_part_copy.sh); then + # shellcheck disable=SC2154 + log 2 "error uploading part $i: $result" + return 1 + fi + log 5 "result: $result" + if [ "$result" != "200" ]; then + log 2 "error uploading part $i: $(cat "$TEST_FILE_FOLDER/$file_name")" + return 1 + fi + if ! etag=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/$file_name" 2>&1); then + log 2 "error retrieving etag: $etag" + return 1 + fi + parts_payload+="$etag$part_number" + done + if ! complete_multipart_upload_rest "$bucket" "$key" "$upload_id" "$parts_payload"; then + log 2 "error completing multipart upload" + return 1 + fi + return 0 +} + +create_upload_finish_wrong_etag() { + if ! check_param_count "create_upload_finish_wrong_etag" "bucket, key" 2 $#; then + return 1 + fi + local bucket="$1" key="$2" + local etag part_number response parts_payload result result_file error_file upload_id + + if ! response=$(get_file_names 2 2>&1); then + log 2 "error getting file names: $response" + return 1 + fi + read -r result_file error_file <<< "$response" + + etag="gibberish" + part_number=1 + if ! response=$(create_multipart_upload_rest "$bucket" "$key" "" "parse_upload_id" 2>&1); then + log 2 "error creating upload and getting ID: $response" + return 1 + fi + upload_id="$response" + + parts_payload="$etag$part_number" + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$bucket" OBJECT_KEY="$key" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/$result_file" ./tests/rest_scripts/complete_multipart_upload.sh); then + log 2 "error completing multipart upload: $result" + return 1 + fi + if [ "$result" != "400" ]; then + log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/$result_file")" + return 1 + fi + if ! get_xml_data "$TEST_FILE_FOLDER/$result_file" "$TEST_FILE_FOLDER/$error_file"; then + log 2 "error getting XML data" + return 1 + fi + if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "InvalidPart" "Code"; then + log 2 "code mismatch" + return 1 + fi + if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "$upload_id" "UploadId"; then + log 2 "upload ID mismatch" + return 1 + fi + if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "$part_number" "PartNumber"; then + log 2 "part number mismatch" + return 1 + fi + if ! check_xml_element "$TEST_FILE_FOLDER/$error_file" "$etag" "ETag"; then + log 2 "etag mismatch" + return 1 + fi + return 0 +} diff --git a/tests/drivers/complete_multipart_upload/complete_multipart_upload_s3api.sh b/tests/drivers/complete_multipart_upload/complete_multipart_upload_s3api.sh new file mode 100644 index 00000000..c96019d0 --- /dev/null +++ b/tests/drivers/complete_multipart_upload/complete_multipart_upload_s3api.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +multipart_upload_s3api_complete_from_bucket() { + if ! check_param_count "multipart_upload_s3api_complete_from_bucket" "bucket, copy source, part count, upload ID" 4 $#; then + return 1 + fi + local bucket="$1" copy_source="$2" part_count="$3" upload_id="$4" + local parts i response quoted_etag error + + parts="[" + for ((i = 1; i <= part_count; i++)); do + if ! response=$(upload_part_copy "$bucket" "${copy_source}-copy" "$upload_id" "$copy_source" "$i" 2>&1); then + log 2 "error uploading part $i: $response" + return 1 + fi + quoted_etag="$response" + # UploadPartCopy etag is returned double quoted, so not necessary to quote it here + parts+="{\"ETag\": $quoted_etag, \"PartNumber\": $i}" + if [[ $i -ne $part_count ]]; then + parts+="," + fi + done + parts+="]" + + if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$bucket" --key "${copy_source}-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}' 2>&1); then + log 2 "Error completing upload: $error" + return 1 + fi + return 0 +} + +multipart_upload_from_bucket() { + if ! check_param_count "multipart_upload_from_bucket" "bucket, copy source, key, part count" 4 $#; then + return 1 + fi + local bucket="$1" copy_source="$2" key="$3" part_count="$4" + local segments upload_id + local -a segment_array + + if ! segments=$(split_file "$key" "$part_count" 2>&1); then + log 2 "error splitting file" + return 1 + fi + read -r -a segment_array <<< "$segments" + + for ((i=0;i&1); then + log 2 "error running first multipart upload: $response" + return 1 + fi + upload_id="$response" + + if ! multipart_upload_s3api_complete_from_bucket "$bucket" "$copy_source" "$part_count" "$upload_id"; then + log 2 " error completing multipart upload from bucket" + return 1 + fi + return 0 +} + +multipart_upload_from_bucket_range() { + if ! check_param_count "multipart_upload_from_bucket_range" "bucket, copy source, key, part count, range" 5 $#; then + return 1 + fi + local bucket="$1" copy_source="$2" key="$3" part_count="$4" range="$5" + local segments parts quoted_etag upload_id + local -a segment_array + + if ! segments=$(split_file "$key" "$part_count" 2>&1); then + log 2 "error splitting file" + return 1 + fi + read -r -a segment_array <<< "$segments" + + for ((i=0;i&1); then + log 2 "error running first multpart upload: $response" + return 1 + fi + upload_id="$response" + + parts="[" + for ((i = 1; i <= part_count; i++)); do + if ! response=$(upload_part_copy_with_range "$bucket" "${copy_source}-copy" "$upload_id" "$copy_source" "$i" "$range" 2>&1); then + # shellcheck disable=SC2154 + log 2 "error uploading part $i: '$response'" + return 1 + fi + quoted_etag="$response" + # UploadPartCopy etag is returned double quoted, so not necessary to quote it here + parts+="{\"ETag\": $quoted_etag, \"PartNumber\": $i}" + if [[ $i -ne $4 ]]; then + parts+="," + fi + done + parts+="]" + if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}'); then + log 2 "Error completing upload: $error" + return 1 + fi + return 0 +} + +multipart_upload_custom() { + if ! check_param_count_gt "bucket, key, file, part count, optional additional parameters" 4 $#; then + return 1 + fi + local bucket="$1" key="$2" file="$3" part_count="$4" params=("${@:5}") + + # shellcheck disable=SC2086 disable=SC2048 + if ! multipart_upload_before_completion_custom "$bucket" "$key" "$file" "$part_count" "${params[@]}"; then + log 2 "error performing pre-completion multipart upload" + return 1 + fi + + log 5 "upload ID: $upload_id, parts: $parts" + if ! complete_multipart_upload "$bucket" "$key" "$upload_id" "$parts"; then + log 2 "Error completing upload" + return 1 + fi + return 0 +} + +multipart_upload() { + if ! check_param_count "multipart_upload" "bucket, key, file, part count" 4 $#; then + return 1 + fi + local bucket="$1" key="$2" file="$3" part_count="$4" + + if ! multipart_upload_before_completion "$bucket" "$key" "$file" "$part_count"; then + log 2 "error performing pre-completion multipart upload" + return 1 + fi + + if ! complete_multipart_upload "$bucket" "$key" "$upload_id" "$parts"; then + log 2 "Error completing upload" + return 1 + fi + return 0 +} + +# perform a multi-part upload +# params: bucket, key, source file location, number of parts +# return 0 for success, 1 for failure +multipart_upload_with_params() { + if ! check_param_count "multipart_upload_with_params" "bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging" 10 $#; then + return 1 + fi + local bucket="$1" + local key="$2" + local file="$3" + local part_count="$4" + local content_type="$5" + local metadata="$6" + local hold_status="$7" + local lock_mode="$8" + local retain_until_date="$9" + local tagging="${10}" + + if ! multipart_upload_before_completion_with_params "$bucket" "$key" "$file" "$part_count" "$content_type" \ + "$metadata" "$hold_status" "$lock_mode" "$retain_until_date" "$tagging"; then + log 2 "error performing pre-completion multipart upload" + return 1 + fi + log 5 "Upload parts: $parts" + + if ! complete_multipart_upload "$bucket" "$key" "$upload_id" "$parts"; then + log 2 "Error completing upload" + return 1 + fi + return 0 +} diff --git a/tests/drivers/create_bucket/create_bucket_s3api.sh b/tests/drivers/create_bucket/create_bucket_s3api.sh new file mode 100644 index 00000000..78535bf6 --- /dev/null +++ b/tests/drivers/create_bucket/create_bucket_s3api.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +setup_multipart_upload_with_params() { + if ! check_param_count "setup_multipart_upload_with_params" "bucket, key" 2 $#; then + return 1 + fi + local bucket="$1" key="$2" + local os_name later_seconds now_epoch later_epoch later result + + os_name="$(uname)" + if [ "$DIRECT" == "true" ]; then + later_seconds="40" + else + later_seconds="20" + fi + + now_epoch=$(date +%s) + later_epoch=$((now_epoch + later_seconds)) + if [[ "$os_name" == "Darwin" ]]; then + later=$(date -r "$later_epoch" -u +"%Y-%m-%dT%H:%M:%SZ") + else + later=$(date -u -d "@$later_epoch" +"%Y-%m-%dT%H:%M:%SZ") + fi + log 5 "later in function: $later" + + if ! result=$(dd if=/dev/urandom of="${TEST_FILE_FOLDER}/${key}" bs=20971520 count=1 2>&1); then + log 2 "error creating large file: $result" + return 1 + fi + + if ! setup_bucket_object_lock_enabled_v2 "$bucket"; then + log 2 "error creating bucket: $bucket" + return 1 + fi + log 5 "later in function: $later" + echo "$later" + return 0 +} diff --git a/tests/drivers/get_object/get_object_s3api.sh b/tests/drivers/get_object/get_object_s3api.sh new file mode 100644 index 00000000..5c621cfb --- /dev/null +++ b/tests/drivers/get_object/get_object_s3api.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +run_and_verify_multipart_upload_with_valid_range() { + if ! check_param_count "run_and_verify_multipart_upload_with_valid_range" "bucket, key, full 5MB file path" 3 $#; then + return 1 + fi + local bucket="$1" key="$2" large_file="$3" + local range_max=$((5*1024*1024-1)) object_size + + if ! multipart_upload_from_bucket_range "$bucket" "$key" "$large_file" 4 "bytes=0-$range_max"; then + log 2 "error with multipart upload" + return 1 + fi + if ! get_object "s3api" "$bucket" "${key}-copy" "${large_file}-copy"; then + log 2 "error getting object" + return 1 + fi + if [[ $(uname) == 'Darwin' ]]; then + object_size=$(stat -f%z "${large_file}-copy") + else + object_size=$(stat --format=%s "${large_file}-copy") + fi + if [[ object_size -ne $((range_max*4+4)) ]]; then + log 2 "object size mismatch ($object_size, $((range_max*4+4)))" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/drivers/head_object/head_object_s3api.sh b/tests/drivers/head_object/head_object_s3api.sh index e8cdb441..873a9ef3 100644 --- a/tests/drivers/head_object/head_object_s3api.sh +++ b/tests/drivers/head_object/head_object_s3api.sh @@ -14,13 +14,38 @@ # specific language governing permissions and limitations # under the License. -get_and_verify_metadata() { - if [ $# -ne 7 ]; then - log 2 "'get_and_verify_metadata' requires bucket file, expected content type, - expected metadata key, expected metadata val, expected hold status, expected retention mode, expected retention date" +get_and_verify_metadata_value() { + if ! check_param_count_v2 "data file, expected metadata key, expected metadata val" 3 $#; then return 1 fi - if ! head_object "s3api" "$BUCKET_ONE_NAME" "$1"; then + local data_file="$1" expected_metadata_key="$2" expected_metadata_val="$3" + local response value + + if ! response=$(jq -r "$expected_metadata_key" - <<< "$data_file" 2>&1); then + log 2 "error retrieving content type: $response" + return 1 + fi + value="$response" + if [[ "$value" != "$expected_metadata_val" ]]; then + log 2 "'$expected_metadata_key' mismatch (expected '$expected_metadata_val', actual '$value')" + return 1 + fi + return 0 +} + +get_and_verify_metadata() { + if ! check_param_count_v2 "bucket file, expected content type, expected metadata key, expected metadata val, expected hold status, expected retention mode, expected retention date" 7 $#; then + return 1 + fi + local bucket_file="$1" + local expected_content_type="$2" + local expected_metadata_key="$3" + local expected_metadata_val="$4" + local expected_hold_status="$5" + local expected_retention_mode="$6" + local expected_retention_date="$7" + + if ! head_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file"; then log 2 "error retrieving metadata" return 1 fi @@ -28,45 +53,24 @@ get_and_verify_metadata() { raw_metadata=$(echo "$metadata" | grep -v "InsecureRequestWarning") log 5 "raw metadata: $raw_metadata" - if ! content_type=$(echo "$raw_metadata" | jq -r ".ContentType" 2>&1); then - log 2 "error retrieving content type: $content_type" + if ! get_and_verify_metadata_value "$raw_metadata" ".ContentType" "$expected_content_type"; then + log 2 "error verifying .ContentType" return 1 fi - if [[ $content_type != "$2" ]]; then - log 2 "content type mismatch ($content_type, $2)" + if ! get_and_verify_metadata_value "$raw_metadata" ".Metadata.${expected_metadata_key}" "$expected_metadata_val"; then + log 2 "error verifying .Metadata.${expected_metadata_key}" return 1 fi - log 5 "metadata key: $3" - if ! meta_val=$(echo "$raw_metadata" | jq -r ".Metadata.$3" 2>&1); then - log 2 "error retrieving metadata val: $meta_val" + if ! get_and_verify_metadata_value "$raw_metadata" ".ObjectLockLegalHoldStatus" "$expected_hold_status"; then + log 2 "error verifying .ObjectLockLegalHoldStatus" return 1 fi - if [[ $meta_val != "$4" ]]; then - log 2 "metadata val mismatch ($meta_val, $4)" + if ! get_and_verify_metadata_value "$raw_metadata" ".ObjectLockMode" "$expected_retention_mode"; then + log 2 "error verifying .ObjectLockLegalHoldStatus" return 1 fi - if ! hold_status=$(echo "$raw_metadata" | jq -r ".ObjectLockLegalHoldStatus" 2>&1); then - log 2 "error retrieving hold status: $hold_status" - return 1 - fi - if [[ $hold_status != "$5" ]]; then - log 2 "hold status mismatch ($hold_status, $5)" - return 1 - fi - if ! retention_mode=$(echo "$raw_metadata" | jq -r ".ObjectLockMode" 2>&1); then - log 2 "error retrieving retention mode: $retention_mode" - return 1 - fi - if [[ $retention_mode != "$6" ]]; then - log 2 "retention mode mismatch ($retention_mode, $6)" - return 1 - fi - if ! retain_until_date=$(echo "$raw_metadata" | jq -r ".ObjectLockRetainUntilDate" 2>&1); then - log 2 "error retrieving retain until date: $retain_until_date" - return 1 - fi - if [[ $retain_until_date != "$7"* ]]; then - log 2"retention date mismatch ($retain_until_date, $7)" + if ! get_and_verify_metadata_value "$raw_metadata" ".ObjectLockRetainUntilDate" "$expected_retention_date"; then + log 2 "error verifying .ObjectLockRetainUntilDate" return 1 fi return 0 diff --git a/tests/drivers/http.sh b/tests/drivers/http.sh new file mode 100644 index 00000000..75ddcc8b --- /dev/null +++ b/tests/drivers/http.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bats + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +check_redirect_response() { + if ! check_param_count_v2 "URL, redirect URL, protocol" 3 $#; then + return 1 + fi + local url="$1" redirect_url="$2" protocol="$3" + local response response_file + + if ! response=$(curl -ksi "${protocol}://${url}" 2>&1); then + log 2 "error sending curl message: $response" + return 1 + fi + if [[ "$response" != *"HTTP/1.1 301"* ]]; then + log 2 "expected 301 response (response: '$response')" + return 1 + fi + if ! response_file=$(get_file_name 2>&1); then + log 2 "error getting file name: $response_file" + return 1 + fi + echo "$response" > "$TEST_FILE_FOLDER/$response_file" + if ! check_header_key_and_value "$TEST_FILE_FOLDER/$response_file" "Location" "${protocol}://$redirect_url/"; then + log 2 "error checking redirect header key and value (data: '$response')" + return 1 + fi + return 0 +} \ No newline at end of file diff --git a/tests/drivers/put_bucket_policy/put_bucket_policy_rest.sh b/tests/drivers/put_bucket_policy/put_bucket_policy_rest.sh index 0767d7e7..2878abba 100644 --- a/tests/drivers/put_bucket_policy/put_bucket_policy_rest.sh +++ b/tests/drivers/put_bucket_policy/put_bucket_policy_rest.sh @@ -51,4 +51,39 @@ put_public_bucket_policy() { return 1 fi return 0 -} \ No newline at end of file +} + +create_website_with_random_string_and_add_permissions() { + if ! check_param_count_v2 "bucket name" 1 $#; then + return 1 + fi + local bucket_name="$1" + local response random_string policy_file + + if ! response=$(create_website_with_random_string "$bucket_name" 2>&1); then + log 2 "error creating website: $response" + return 1 + fi + read -r _ random_string <<< "$response" + log 5 "random string: $random_string" + + if [ "$DIRECT" == "true" ]; then + if ! put_public_access_block "$bucket_name" "BlockPublicAcls=false,IgnorePublicAcls=false,BlockPublicPolicy=false,RestrictPublicBuckets=false" > /dev/null 2>&1; then + log 2 "error putting public access block" + return 1 + fi + fi + + if ! response=$(setup_policy_with_single_statement_v2 "2012-10-17" "Allow" "*" "s3:GetObject" "arn:aws:s3:::$bucket_name/*" 2>&1); then + log 2 "error setting up policy: $response" + return 1 + fi + policy_file="$response" + + if ! put_bucket_policy_rest "$bucket_name" "$TEST_FILE_FOLDER"/"$policy_file"; then + log 2 "error putting bucket policy" + return 1 + fi + printf '%s\n' "$random_string" + return 0 +} diff --git a/tests/drivers/put_bucket_website/put_bucket_website_rest.sh b/tests/drivers/put_bucket_website/put_bucket_website_rest.sh index 7cb9ce27..43cff1e5 100644 --- a/tests/drivers/put_bucket_website/put_bucket_website_rest.sh +++ b/tests/drivers/put_bucket_website/put_bucket_website_rest.sh @@ -15,6 +15,7 @@ # under the License. source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/put_bucket_policy/put_bucket_policy_rest.sh source ./tests/drivers/string.sh create_website_with_random_string() { @@ -37,4 +38,25 @@ create_website_with_random_string() { fi printf '%s\n' "$file_name $test_string" return 0 +} + +put_redirect() { + if ! check_param_count_ge_le "bucket name, redirect URL, protocol (optional)" 2 3 $#; then + return 1 + fi + local bucket_name="$1" redirect_url="$2" protocol="$3" + local json_payload + + if [[ "$protocol" == "https" ]] || [[ "$protocol" == "http" ]]; then + json_payload="{\"RedirectAllRequestsTo\":{\"HostName\":\"$redirect_url\",\"Protocol\":\"$protocol\"}}" + else + json_payload="{\"RedirectAllRequestsTo\":{\"HostName\":\"$redirect_url\"}}" + fi + + if ! response=$(send_rest_go_command "200" "-commandType" "putBucketWebsiteConfiguration" "-bucketName" "$bucket_name" \ + "-websiteConfiguration" "$json_payload" 2>&1); then + log 2 "error putting website configuration: $response" + return 1 + fi + return 0 } \ No newline at end of file diff --git a/tests/drivers/put_object/put_object_s3api.sh b/tests/drivers/put_object/put_object_s3api.sh new file mode 100644 index 00000000..1ec68dad --- /dev/null +++ b/tests/drivers/put_object/put_object_s3api.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +split_and_put_file() { + if ! check_param_count_v2 "bucket, key, copy source, part count" 4 $#; then + return 1 + fi + local bucket="$1" key="$2" copy_source="$3" part_count="$4" + local file_parts idx + local -a part_array + + if ! file_parts=$(split_file "$copy_source" "$part_count" 2>&1); then + log 2 "error splitting file: $file_parts" + return 1 + fi + read -r -a part_array <<< "$file_parts" + + for ((idx=0;idx&1); then - printf "error installing go dependencies: %s\n" "$response" - exit 1 + if [ -n "$INSTALL_ONLY_FOLDER" ]; then + if ! response=$(GOPROXY=off go mod download 2>&1); then + printf "error installing go dependencies: %s\n" "$response" + exit 1 + fi + else + if ! response=$(go mod download 2>&1); then + printf "error installing go dependencies: %s\n" "$response" + exit 1 + fi fi fi diff --git a/tests/test_rest_get_object.sh b/tests/test_rest_get_object.sh index 0788fcc7..c6133dd4 100755 --- a/tests/test_rest_get_object.sh +++ b/tests/test_rest_get_object.sh @@ -20,6 +20,7 @@ load ./bats-assert/load source ./tests/commands/get_object.sh source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh source ./tests/setup.sh # tags: curl,GetObject,HeadObject,range diff --git a/tests/test_rest_head_object.sh b/tests/test_rest_head_object.sh index 5b4bb1a5..2758b9db 100755 --- a/tests/test_rest_head_object.sh +++ b/tests/test_rest_head_object.sh @@ -19,6 +19,7 @@ load ./bats-assert/load source ./tests/setup.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh source ./tests/drivers/get_object_attributes/get_object_attributes_rest.sh source ./tests/drivers/string.sh diff --git a/tests/test_rest_list_parts.sh b/tests/test_rest_list_parts.sh index 0e8bf9f9..46fad97b 100755 --- a/tests/test_rest_list_parts.sh +++ b/tests/test_rest_list_parts.sh @@ -18,6 +18,7 @@ load ./bats-support/load load ./bats-assert/load source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh source ./tests/setup.sh @test "REST - ListParts - invalid part number marker" { diff --git a/tests/test_rest_multipart.sh b/tests/test_rest_multipart.sh index 19f90c08..27bb5ec1 100755 --- a/tests/test_rest_multipart.sh +++ b/tests/test_rest_multipart.sh @@ -21,8 +21,10 @@ source ./tests/setup.sh source ./tests/drivers/file.sh source ./tests/drivers/rest.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_rest.sh source ./tests/drivers/list_buckets/list_buckets_rest.sh +source ./tests/drivers/put_object/put_object_s3api.sh source ./tests/drivers/upload_part/upload_part_rest.sh # tags: curl,multipart,CreateMultipartUpload,AbortMultipartUpload diff --git a/tests/test_rest_website.sh b/tests/test_rest_website.sh index c309f9fb..904c9802 100755 --- a/tests/test_rest_website.sh +++ b/tests/test_rest_website.sh @@ -22,6 +22,7 @@ source ./tests/drivers/create_bucket/create_bucket_rest.sh source ./tests/drivers/get_bucket_website/get_bucket_website_rest.sh source ./tests/drivers/put_bucket_website/put_bucket_website_rest.sh source ./tests/drivers/cloudfront.sh +source ./tests/drivers/http.sh source ./tests/drivers/string.sh source ./tests/setup_common.sh @@ -35,6 +36,10 @@ setup() { setup_versitygw_for_website() { optional_params=("$@") + if [ "$DIRECT" == "true" ]; then + return 0 + fi + run setup_versitygw "${optional_params[@]}" assert_success read -r process_id process_id_two <<< "$output" @@ -42,6 +47,7 @@ setup_versitygw_for_website() { if [ -n "$process_id_two" ]; then export VERSITYGW_PID_2="$process_id_two" fi + return 0 } teardown() { @@ -154,20 +160,39 @@ teardown() { assert_success } -@test "REST - GetBucketWebsite - no HTTPS" { - local bucket_name random_string +@test "REST - GetBucketWebsite - redirect" { + local bucket_one_name bucket_two_name random_string protocol="http" setup_versitygw_for_website "--website-no-tls" - run setup_bucket_v3 "$BUCKET_ONE_NAME" + run setup_buckets_v3 "$BUCKET_ONE_NAME" "$BUCKET_TWO_NAME" assert_success - bucket_name="$output" + read -r bucket_one_name bucket_two_name <<< "$output" - run create_website_with_random_string_and_add_permissions "$bucket_name" + run create_website_with_random_string_and_add_permissions "$bucket_one_name" assert_success read -r random_string <<< "$output" + log 5 "random string after output: $random_string" - run curl -ks "http://${bucket_name}.${WEBSITE_DOMAIN}${WEBSITE}" + if [ "$DIRECT" == "true" ]; then + domain_name="${bucket_one_name}.s3-website.${AWS_REGION}.amazonaws.com" + else + domain_name="${bucket_one_name}.${WEBSITE_DOMAIN}${WEBSITE}" + fi + + run put_redirect "$bucket_two_name" "${domain_name}" "$protocol" + assert_success + + if [ "$DIRECT" == "true" ]; then + domain_two_name="${bucket_two_name}.s3-website.${AWS_REGION}.amazonaws.com" + else + domain_two_name="${bucket_two_name}.${WEBSITE_DOMAIN}${WEBSITE}" + fi + + run check_redirect_response "$domain_two_name" "$domain_name" "$protocol" + assert_success + + run curl -ksL "${protocol}://${domain_two_name}" assert_success assert_output "$random_string" } diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index d79a744c..10615432 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -23,13 +23,16 @@ source ./tests/commands/get_object.sh source ./tests/commands/list_multipart_uploads.sh source ./tests/commands/put_object.sh source ./tests/drivers/file.sh -source ./tests/drivers/head_object/head_object_s3api.sh +source ./tests/drivers/complete_multipart_upload/complete_multipart_upload_s3api.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/create_bucket/create_bucket_s3api.sh +source ./tests/drivers/get_object/get_object_s3api.sh source ./tests/drivers/get_object_legal_hold/get_object_legal_hold.sh source ./tests/drivers/get_object_tagging/get_object_tagging.sh +source ./tests/drivers/head_object/head_object_s3api.sh source ./tests/drivers/list_multipart_uploads/list_multipart_uploads_s3api.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh -source ./tests/util/util_multipart.sh +source ./tests/drivers/put_object/put_object_s3api.sh source ./tests/util/util_multipart_abort.sh source ./tests/util/util_multipart_before_completion.sh @@ -91,6 +94,8 @@ export RUN_USERS=true "$expected_tag_key=$expected_tag_val" assert_success + log 5 "later: $later" + later="${later/Z/+00:00}" run get_and_verify_metadata "$bucket_file" "$expected_content_type" "$expected_meta_key" "$expected_meta_val" \ "$expected_hold_status" "$expected_retention_mode" "$later" assert_success diff --git a/tests/test_s3api_policy.sh b/tests/test_s3api_policy.sh index 470a8e34..82986363 100755 --- a/tests/test_s3api_policy.sh +++ b/tests/test_s3api_policy.sh @@ -22,7 +22,6 @@ source ./tests/setup.sh source ./tests/test_s3api_policy_bucket.sh source ./tests/test_s3api_policy_multipart.sh source ./tests/test_s3api_policy_object.sh -source ./tests/util/util_multipart.sh source ./tests/util/util_multipart_abort.sh source ./tests/util/util_multipart_before_completion.sh source ./tests/util/util_policy.sh diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh deleted file mode 100644 index b5f79d8b..00000000 --- a/tests/util/util_multipart.sh +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2024 Versity Software -# This file is licensed under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source ./tests/commands/put_object.sh -source ./tests/drivers/create_multipart_upload/create_multipart_upload_rest.sh - -multipart_upload_s3api_complete_from_bucket() { - if ! check_param_count "multipart_upload_s3api_complete_from_bucket" "bucket, copy source, part count" 3 $#; then - return 1 - fi - parts="[" - for ((i = 1; i <= $3; i++)); do - # shellcheck disable=SC2154 - if ! upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i"; then - log 2 "error uploading part $i" - return 1 - fi - # shellcheck disable=SC2154 - parts+="{\"ETag\": $etag, \"PartNumber\": $i}" - if [[ $i -ne $3 ]]; then - parts+="," - fi - done - parts+="]" - - if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}' 2>&1); then - log 2 "Error completing upload: $error" - return 1 - fi - return 0 -} - -multipart_upload_from_bucket() { - if ! check_param_count "multipart_upload_from_bucket" "bucket, copy source, key, part count" 4 $#; then - return 1 - fi - - if ! segments=$(split_file "$3" "$4" 2>&1); then - log 2 "error splitting file" - return 1 - fi - read -r -a segment_array <<< "$segments" - - for ((i=0;i<$4;i++)) { - log 5 "key: $3" - if ! put_object "s3api" "${segment_array[$i]}" "$1" "$2-$i"; then - log 2 "error copying object" - return 1 - fi - } - - local response - if ! response=$(create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id" 2>&1); then - log 2 "error running first multipart upload: $response" - return 1 - fi - upload_id="$response" - - if ! multipart_upload_s3api_complete_from_bucket "$1" "$2" "$4"; then - log 2 " error completing multipart upload from bucket" - return 1 - fi - return 0 -} - -split_and_put_file() { - if ! check_param_count "split_and_put_file" "bucket, key, copy source, part count" 4 $#; then - return 1 - fi - if ! file_parts=$(split_file "$3" "$4" 2>&1); then - log 2 "error splitting file: $file_parts" - return 1 - fi - read -r -a part_array <<< "$file_parts" - - for ((i=0;i<$4;i++)) { - log 5 "key: $2, file info: $(ls -l "${part_array[$i]}")" - if ! put_object "s3api" "${part_array[$i]}" "$1" "$2-$i"; then - log 2 "error copying object" - return 1 - fi - } - return 0 -} - -multipart_upload_from_bucket_range() { - if ! check_param_count "multipart_upload_from_bucket_range" "bucket, copy source, key, part count, range" 5 $#; then - return 1 - fi - if ! segments=$(split_file "$3" "$4" 2>&1); then - log 2 "error splitting file" - return 1 - fi - read -r -a segment_array <<< "$segments" - - for ((i=0;i<$4;i++)) { - log 5 "key: $3, file info: $(ls -l "$3"-"$i")" - if ! put_object "s3api" "${segment_array[$i]}" "$1" "$2-$i"; then - log 2 "error copying object" - return 1 - fi - } - - local response - if ! response=$(create_multipart_upload_rest "$1" "$2-copy" "" "parse_upload_id" 2>&1); then - log 2 "error running first multpart upload: $response" - return 1 - fi - upload_id="$response" - - parts="[" - for ((i = 1; i <= $4; i++)); do - if ! upload_part_copy_with_range "$1" "$2-copy" "$upload_id" "$2" "$i" "$5"; then - # shellcheck disable=SC2154 - log 2 "error uploading part $i: $upload_part_copy_error" - return 1 - fi - parts+="{\"ETag\": $etag, \"PartNumber\": $i}" - if [[ $i -ne $4 ]]; then - parts+="," - fi - done - parts+="]" - if ! error=$(aws --no-verify-ssl s3api complete-multipart-upload --bucket "$1" --key "$2-copy" --upload-id "$upload_id" --multipart-upload '{"Parts": '"$parts"'}'); then - log 2 "Error completing upload: $error" - return 1 - fi - return 0 -} - -multipart_upload_custom() { - if ! check_param_count_gt "bucket, key, file, part count, optional additional parameters" 4 $$; then - return 1 - fi - - # shellcheck disable=SC2086 disable=SC2048 - if ! multipart_upload_before_completion_custom "$1" "$2" "$3" "$4" ${*:5}; then - log 2 "error performing pre-completion multipart upload" - return 1 - fi - - log 5 "upload ID: $upload_id, parts: $parts" - if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then - log 2 "Error completing upload" - return 1 - fi - return 0 -} - -multipart_upload() { - if ! check_param_count "multipart_upload" "bucket, key, file, part count" 4 $#; then - return 1 - fi - - if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then - log 2 "error performing pre-completion multipart upload" - return 1 - fi - - if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then - log 2 "Error completing upload" - return 1 - fi - return 0 -} - -# perform a multi-part upload -# params: bucket, key, source file location, number of parts -# return 0 for success, 1 for failure -multipart_upload_with_params() { - if ! check_param_count "multipart_upload_with_params" "bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging" 10 $#; then - return 1 - fi - log 5 "1: $1, 2: $2, 3: $3, 4: $4, 5: $5, 6: $6, 7: $7, 8: $8, 9: $9, 10: ${10}" - - if ! multipart_upload_before_completion_with_params "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}"; then - log 2 "error performing pre-completion multipart upload" - return 1 - fi - log 5 "Upload parts: $parts" - - if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then - log 2 "Error completing upload" - return 1 - fi - return 0 -} - -run_and_verify_multipart_upload_with_valid_range() { - if ! check_param_count "run_and_verify_multipart_upload_with_valid_range" "bucket, key, 5MB file" 3 $#; then - return 1 - fi - range_max=$((5*1024*1024-1)) - if ! multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-$range_max"; then - log 2 "error with multipart upload" - return 1 - fi - if ! get_object "s3api" "$1" "$2-copy" "$3-copy"; then - log 2 "error getting object" - return 1 - fi - if [[ $(uname) == 'Darwin' ]]; then - object_size=$(stat -f%z "$3-copy") - else - object_size=$(stat --format=%s "$3-copy") - fi - if [[ object_size -ne $((range_max*4+4)) ]]; then - log 2 "object size mismatch ($object_size, $((range_max*4+4)))" - return 1 - fi - return 0 -} - -create_upload_part_copy_rest() { - if ! check_param_count "create_upload_part_copy_rest" "bucket, key, >20MB file" 3 $#; then - return 1 - fi - if ! split_and_put_file "$1" "$2" "$3" 4; then - log 2 "error splitting and putting file" - return 1 - fi - - local response - if ! response=$(create_multipart_upload_rest "$1" "$2" "" "parse_upload_id" 2>&1); then - log 2 "error creating upload and getting ID: $response" - return 1 - fi - upload_id="$response" - - parts_payload="" - for ((i=0; i<=3; i++)); do - part_number=$((i+1)) - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" PART_NUMBER="$part_number" UPLOAD_ID="$upload_id" PART_LOCATION="$1/$2-$i" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/upload_part_copy.sh); then - # shellcheck disable=SC2154 - log 2 "error uploading part $i: $result" - return 1 - fi - log 5 "result: $result" - if [ "$result" != "200" ]; then - log 2 "error uploading part $i: $(cat "$TEST_FILE_FOLDER/response.txt")" - return 1 - fi - if ! etag=$(xmllint --xpath '//*[local-name()="ETag"]/text()' "$TEST_FILE_FOLDER/response.txt" 2>&1); then - log 2 "error retrieving etag: $etag" - return 1 - fi - parts_payload+="$etag$part_number" - done - if ! complete_multipart_upload_rest "$1" "$2" "$upload_id" "$parts_payload"; then - log 2 "error completing multipart upload" - return 1 - fi - return 0 -} - -create_upload_finish_wrong_etag() { - if ! check_param_count "create_upload_finish_wrong_etag" "bucket, key" 2 $#; then - return 1 - fi - - etag="gibberish" - part_number=1 - if ! create_multipart_upload_rest "$1" "$2" "" "parse_upload_id"; then - log 2 "error creating upload and getting ID" - return 1 - fi - parts_payload="$etag$part_number" - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" UPLOAD_ID="$upload_id" PARTS="$parts_payload" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/complete_multipart_upload.sh); then - log 2 "error completing multipart upload: $result" - return 1 - fi - if [ "$result" != "400" ]; then - log 2 "complete multipart upload returned code $result: $(cat "$TEST_FILE_FOLDER/result.txt")" - return 1 - fi - if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error.txt"; then - log 2 "error getting XML data" - return 1 - fi - if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "InvalidPart" "Code"; then - log 2 "code mismatch" - return 1 - fi - if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "$upload_id" "UploadId"; then - log 2 "upload ID mismatch" - return 1 - fi - if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "$part_number" "PartNumber"; then - log 2 "part number mismatch" - return 1 - fi - if ! check_xml_element "$TEST_FILE_FOLDER/error.txt" "$etag" "ETag"; then - log 2 "etag mismatch" - return 1 - fi - return 0 -} - -setup_multipart_upload_with_params() { - if ! check_param_count "setup_multipart_upload_with_params" "bucket, key" 2 $#; then - return 1 - fi - os_name="$(uname)" - if [ "$DIRECT" == "true" ]; then - later_seconds="40" - else - later_seconds="20" - fi - if [[ "$os_name" == "Darwin" ]]; then - now=$(date -u +"%Y-%m-%dT%H:%M:%S") - later=$(date -j -v "+${later_seconds}S" -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S") - else - now=$(date +"%Y-%m-%dT%H:%M:%S") - later=$(date -d "$now $later_seconds seconds" +"%Y-%m-%dT%H:%M:%S") - fi - log 5 "later in function: $later" - - if ! create_test_files "$2"; then - log 2 "error creating test file" - return 1 - fi - - if ! result=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER/$2" bs=20M count=1 2>&1); then - log 2 "error creating large file: $result" - return 1 - fi - - if ! bucket_cleanup_if_bucket_exists "$BUCKET_ONE_NAME"; then - log 2 "error cleaning up bucket" - return 1 - fi - # in static bucket config, bucket will still exist - if ! bucket_exists "$BUCKET_ONE_NAME"; then - if ! create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"; then - log 2 "error creating bucket with object lock enabled" - return 1 - fi - fi - log 5 "later in function: $later" - echo "$later" - return 0 -} diff --git a/tests/util/util_multipart_before_completion.sh b/tests/util/util_multipart_before_completion.sh index 9ce01ed5..d2d8e4a1 100644 --- a/tests/util/util_multipart_before_completion.sh +++ b/tests/util/util_multipart_before_completion.sh @@ -339,7 +339,6 @@ multipart_upload_before_completion_custom() { fi done parts+="]" - export parts } @@ -347,14 +346,16 @@ multipart_upload_range_too_large() { if ! check_param_count_v2 "bucket, key, file location" 3 $#; then return 1 fi - if multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-1000000000"; then - log 2 "multipart upload succeeded despite overly large range" + local bucket="$1" key="$2" file_location="$3" + local response + + if response=$(multipart_upload_from_bucket_range "$bucket" "$key" "$file_location" 4 "bytes=0-1000000000" 2>&1); then + log 2 "multipart upload succeeded despite overly large range: $response" return 1 fi # shellcheck disable=SC2154 - log 5 "error: $upload_part_copy_error" - if [[ $upload_part_copy_error != *"Range specified is not valid"* ]] && [[ $upload_part_copy_error != *"InvalidRange"* ]]; then - log 2 "unexpected error: $upload_part_copy_error" + if [[ "$response" != *"Range specified is not valid"* ]] && [[ "$response" != *"InvalidRange"* ]]; then + log 2 "unexpected error: $response" return 1 fi return 0 diff --git a/tests/util/util_object.sh b/tests/util/util_object.sh index 599217e5..48483cf6 100644 --- a/tests/util/util_object.sh +++ b/tests/util/util_object.sh @@ -14,7 +14,6 @@ # specific language governing permissions and limitations # under the License. -source ./tests/util/util_multipart.sh source ./tests/util/util_versioning.sh source ./tests/logger.sh source ./tests/commands/abort_multipart_upload.sh