From 9fb039b878dc5ef757bc469dc94ffdd819ddce4f Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Mon, 8 Dec 2025 18:54:48 -0300 Subject: [PATCH] test: util cleanup --- tests/commands/create_bucket.sh | 41 +++ tests/drivers/create_bucket/create_bucket.sh | 35 ++ tests/drivers/file.sh | 28 ++ .../get_bucket_location.sh} | 18 +- .../put_bucket_ownership_controls_rest.sh | 12 + tests/drivers/put_object/put_object_rest.sh | 275 ++++++++++++++++ tests/test_common.sh | 1 - tests/test_mc.sh | 3 +- tests/test_rest.sh | 1 - tests/test_rest_chunked.sh | 1 - tests/test_s3api_bucket.sh | 3 +- tests/test_s3api_object.sh | 1 - tests/test_s3cmd.sh | 2 +- tests/test_user_common.sh | 1 - tests/util/util_bucket_location.sh | 18 - tests/util/util_chunked_upload.sh | 310 ------------------ tests/util/util_create_bucket.sh | 77 ----- tests/util/util_object.sh | 1 - 18 files changed, 407 insertions(+), 421 deletions(-) create mode 100644 tests/drivers/create_bucket/create_bucket.sh rename tests/{util/util_config.sh => drivers/get_bucket_location/get_bucket_location.sh} (61%) delete mode 100644 tests/util/util_bucket_location.sh delete mode 100644 tests/util/util_chunked_upload.sh delete mode 100644 tests/util/util_create_bucket.sh diff --git a/tests/commands/create_bucket.sh b/tests/commands/create_bucket.sh index e65f38e..8d8bc97 100644 --- a/tests/commands/create_bucket.sh +++ b/tests/commands/create_bucket.sh @@ -48,6 +48,31 @@ create_bucket() { return 0 } +create_bucket_invalid_name() { + if [ $# -ne 1 ]; then + log 2 "create bucket w/invalid name missing command type" + return 1 + fi + local exit_code=0 + if [[ $1 == "aws" ]] || [[ $1 == 's3' ]]; then + bucket_create_error=$(aws --no-verify-ssl s3 mb "s3://" 2>&1) || exit_code=$? + elif [[ $1 == 's3api' ]]; then + bucket_create_error=$(aws --no-verify-ssl s3api create-bucket --bucket "s3://" 2>&1) || exit_code=$? + elif [[ $1 == 's3cmd' ]]; then + bucket_create_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb "s3://" 2>&1) || exit_code=$? + elif [[ $1 == 'mc' ]]; then + bucket_create_error=$(mc --insecure mb "$MC_ALIAS/." 2>&1) || exit_code=$? + else + log 2 "invalid command type $1" + return 1 + fi + if [ $exit_code -eq 0 ]; then + log 2 "error: bucket should have not been created but was" + return 1 + fi + echo "$bucket_create_error" +} + create_bucket_with_user() { log 6 "create_bucket_with_user" if ! check_param_count "create_bucket_with_user" "command type, bucket, access ID, secret key" 4 $#; then @@ -125,3 +150,19 @@ create_bucket_rest_expect_success() { fi return 0 } + +create_bucket_rest() { + if ! check_param_count "create_bucket_rest" "bucket name" 1 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/create_bucket.sh 2>&1); then + log 2 "error creating bucket: $result" + return 1 + fi + if [ "$result" != "200" ]; then + bucket_create_error="$(cat "$TEST_FILE_FOLDER/result.txt")" + log 2 "expected '200', was '$result' ($bucket_create_error)" + return 1 + fi + return 0 +} diff --git a/tests/drivers/create_bucket/create_bucket.sh b/tests/drivers/create_bucket/create_bucket.sh new file mode 100644 index 0000000..1e850d7 --- /dev/null +++ b/tests/drivers/create_bucket/create_bucket.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright 2025 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/util/util_mc.sh +source ./tests/logger.sh + +create_and_check_bucket_invalid_name() { + if ! check_param_count_v2 "client" 1 $#; then + return 1 + fi + if ! create_bucket_invalid_name "$1"; then + log 2 "error creating bucket with invalid name" + return 1 + fi + + # shellcheck disable=SC2154 + if [[ "$bucket_create_error" != *"Invalid bucket name "* ]] && [[ "$bucket_create_error" != *"Bucket name cannot"* ]]; then + log 2 "unexpected error: $bucket_create_error" + return 1 + fi + return 0 +} diff --git a/tests/drivers/file.sh b/tests/drivers/file.sh index 3c8f316..d8d09a8 100644 --- a/tests/drivers/file.sh +++ b/tests/drivers/file.sh @@ -123,3 +123,31 @@ setup_bucket_and_large_file_v2() { fi return 0 } + +chunked_upload_trailer_success() { + if ! check_param_count_v2 "checksum" 1 $#; then + return 1 + fi + if ! bucket_name=$(get_bucket_name "$BUCKET_ONE_NAME" 2>&1); then + log 2 "error getting bucket name: $bucket_name" + return 1 + fi + if ! setup_bucket "$bucket_name"; then + log 2 "error setting up bucket" + return 1 + fi + test_file="test-file" + if ! create_test_file "$test_file" 10000; then + log 2 "error creating test file" + return 1 + fi + if ! put_object_chunked_trailer_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$1"; then + log 2 "error performing chunked upload w/trailer" + return 1 + fi + if ! download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"; then + log 2 "error downloading and comparing file" + return 1 + fi + return 0 +} diff --git a/tests/util/util_config.sh b/tests/drivers/get_bucket_location/get_bucket_location.sh similarity index 61% rename from tests/util/util_config.sh rename to tests/drivers/get_bucket_location/get_bucket_location.sh index a3b6c74..611b937 100644 --- a/tests/util/util_config.sh +++ b/tests/drivers/get_bucket_location/get_bucket_location.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright 2024 Versity Software +# Copyright 2025 Versity Software # This file is licensed under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at @@ -14,14 +14,18 @@ # specific language governing permissions and limitations # under the License. -setup_two_buckets() { - setup_bucket "$BUCKET_ONE_NAME" || local setup_result_one=$? - if [[ $setup_result_one -eq 0 ]]; then +get_check_bucket_location_various() { + if ! check_param_count_v2 "client, bucket" 2 $#; then return 1 fi - setup_bucket "$BUCKET_TWO_NAME" || local setup_result_two=$? - if [[ $setup_result_two -eq 0 ]]; then + if ! get_bucket_location "$1" "$2"; then + log 2 "error getting bucket location" + return 1 + fi + # shellcheck disable=SC2154 + if [[ $bucket_location != "null" ]] && [[ $bucket_location != "us-east-1" ]]; then + log 2 "wrong location: '$bucket_location'" return 1 fi return 0 -} \ No newline at end of file +} diff --git a/tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh b/tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh index a6d4099..4616291 100644 --- a/tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh +++ b/tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh @@ -72,3 +72,15 @@ setup_bucket() { fi return 0 } + +setup_two_buckets() { + setup_bucket "$BUCKET_ONE_NAME" || local setup_result_one=$? + if [[ $setup_result_one -eq 0 ]]; then + return 1 + fi + setup_bucket "$BUCKET_TWO_NAME" || local setup_result_two=$? + if [[ $setup_result_two -eq 0 ]]; then + return 1 + fi + return 0 +} diff --git a/tests/drivers/put_object/put_object_rest.sh b/tests/drivers/put_object/put_object_rest.sh index f3e6e1f..2bb67e9 100644 --- a/tests/drivers/put_object/put_object_rest.sh +++ b/tests/drivers/put_object/put_object_rest.sh @@ -14,6 +14,9 @@ # specific language governing permissions and limitations # under the License. +source ./tests/drivers/rest.sh +source ./tests/drivers/openssl.sh + put_object_success_or_access_denied() { if ! check_param_count_v2 "username, password, data file, bucket, key, expect success" 6 $#; then return 1 @@ -94,3 +97,275 @@ put_bucket_object_run_command_expect_error() { fi return 0 } + +attempt_seed_signature_without_content_length() { + if ! check_param_count_v2 "bucket, key, data file" 3 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" NO_CONTENT_LENGTH="true" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 411; then + log 2 "error in sending or checking response code" + return 1 + fi + return 0 +} + +attempt_chunked_upload_with_bad_first_signature() { + if ! check_param_count_v2 "data file, bucket name, key" 3 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" FIRST_SIGNATURE="xxxxxxxx" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + + if ! result=$(send_via_openssl "$TEST_FILE_FOLDER/command.txt"); then + log 2 "error sending command via openssl" + return 1 + fi + log 5 "result: $result" + echo -n "$result" > "$TEST_FILE_FOLDER/result.txt" + if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error_data.txt"; then + log 2 "error parsing XML data from result" + return 1 + fi + response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')" + if ! check_rest_expected_error "$response_code" "$TEST_FILE_FOLDER/error_data.txt" "403" "SignatureDoesNotMatch" "does not match"; then + log 2 "error checking expected REST error" + return 1 + fi + return 0 +} + +chunked_upload_success() { + if ! check_param_count_v2 "data file, bucket name, key" 3 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + + if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then + log 2 "error sending command via openssl or checking response code" + return 1 + fi + return 0 +} + +attempt_chunked_upload_with_bad_final_signature() { + if ! check_param_count_v2 "data file, bucket name, key" 3 $#; then + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" \ + CHUNK_SIZE=8192 \ + TEST_MODE=false \ + COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" \ + FINAL_SIGNATURE="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + if ! result=$(send_via_openssl "$TEST_FILE_FOLDER/command.txt"); then + log 2 "error sending command via openssl" + return 1 + fi + log 5 "response: $result" + echo -n "$result" > "$TEST_FILE_FOLDER/result.txt" + if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error_data.txt"; then + log 2 "error parsing XML data from result" + return 1 + fi + log 5 "xml data: $(cat "$TEST_FILE_FOLDER/error_data.txt")" + response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')" + if ! check_rest_expected_error "$response_code" "$TEST_FILE_FOLDER/error_data.txt" "403" "SignatureDoesNotMatch" "does not match"; then + log 2 "error checking expected REST error" + return 1 + fi + return 0 +} + +put_object_chunked_trailer_success() { + if ! check_param_count_v2 "data file, bucket name, key, checksum type" 4 $#; then + return 1 + fi + # shellcheck disable=SC2097,SC2098 + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false TRAILER="x-amz-checksum-$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + + if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then + log 2 "error sending command via openssl or checking response code" + return 1 + fi + return 0 +} + +put_chunked_upload_trailer_invalid() { + if ! check_param_count_v2 "data file, bucket name, key" 3 $#; then + return 1 + fi + # shellcheck disable=SC2097,SC2098 + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false \ + TRAILER="x-amz-checksum-sha10" \ + INVALID_CHECKSUM_TYPE="true" CHECKSUM="abc" \ + TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + + if ! result=$(send_via_openssl "$TEST_FILE_FOLDER/command.txt"); then + log 2 "error sending command via openssl" + return 1 + fi + response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')" + if [ "$response_code" != "400" ]; then + log 2 "expected response '400', was '$response_code'" + return 1 + fi + error_data="$(echo "$result" | grep "" | sed 's/---//g')" + echo -n "$error_data" > "$TEST_FILE_FOLDER/error-data.txt" + if ! check_xml_error_contains "$TEST_FILE_FOLDER/error-data.txt" "InvalidRequest" "The value specified in the x-amz-trailer header is not supported"; then + log 2 "error checking xml error, message" + return 1 + fi + return 0 +} + +chunked_upload_trailer_invalid_checksum() { + if ! check_param_count_v2 "checksum" 1 $#; then + return 1 + fi + if ! bucket_name=$(get_bucket_name "$BUCKET_ONE_NAME" 2>&1); then + log 2 "error getting bucket name: $bucket_name" + return 1 + fi + if ! setup_bucket "$bucket_name"; then + log 2 "error setting up bucket" + return 1 + fi + test_file="test-file" + if ! create_test_file "$test_file" 10000; then + log 2 "error creating test file" + return 1 + fi + # shellcheck disable=SC2097,SC2098 + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$TEST_FILE_FOLDER/$test_file" \ + BUCKET_NAME="$bucket_name" \ + OBJECT_KEY="$test_file" CHUNK_SIZE=8192 TEST_MODE=false TRAILER="x-amz-checksum-$1" CHECKSUM="a" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + if ! send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/command.txt" "400" "InvalidRequest" "Value for x-amz-checksum-$1 trailing header is invalid."; then + log 2 "error sending openssl and checking response" + return 1 + fi + return 0 +} + +chunked_upload_trailer_incorrect_checksum() { + if ! check_param_count_v2 "checksum" 1 $#; then + return 1 + fi + if ! bucket_name=$(get_bucket_name "$BUCKET_ONE_NAME" 2>&1); then + log 2 "error getting bucket name: $bucket_name" + return 1 + fi + if ! setup_bucket "$bucket_name"; then + log 2 "error setting up bucket" + return 1 + fi + test_file="test-file" + if ! create_test_file "$test_file" 10000; then + log 2 "error creating test file" + return 1 + fi + if ! checksum=$(calculate_incorrect_checksum "$1" "$TEST_FILE_FOLDER/$test_file"); then + log 2 "error calculating incorrect checksum" + return 1 + fi + # shellcheck disable=SC2097,SC2098 + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$TEST_FILE_FOLDER/$test_file" \ + BUCKET_NAME="$bucket_name" \ + OBJECT_KEY="$test_file" CHUNK_SIZE=8192 TEST_MODE=false TRAILER="x-amz-checksum-$1" CHECKSUM="$checksum" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + uppercase_type="$(echo "$1" | tr '[:lower:]' '[:upper:]')" + if ! send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/command.txt" "400" "BadDigest" "The $uppercase_type you specified did not match the calculated checksum."; then + log 2 "error sending openssl and checking response" + return 1 + fi + return 0 +} + +chunked_upload_trailer_different_chunk_size() { + if ! check_param_count_v2 "data file, bucket, key, checksum type" 4 $#; then + return 1 + fi + # shellcheck disable=SC2097,SC2098 + if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ + DATA_FILE="$1" \ + BUCKET_NAME="$2" \ + OBJECT_KEY="$3" CHUNK_SIZE=16384 TEST_MODE=false TRAILER="x-amz-checksum-$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then + log 2 "error creating command: $result" + return 1 + fi + + if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then + log 2 "error sending command via openssl or checking response code" + return 1 + fi + return 0 +} + diff --git a/tests/test_common.sh b/tests/test_common.sh index e8d970c..7067b97 100644 --- a/tests/test_common.sh +++ b/tests/test_common.sh @@ -31,7 +31,6 @@ source ./tests/commands/put_object.sh source ./tests/commands/put_public_access_block.sh source ./tests/drivers/file.sh source ./tests/drivers/params.sh -source ./tests/util/util_bucket_location.sh source ./tests/util/util_file.sh source ./tests/util/util_list_buckets.sh source ./tests/util/util_object.sh diff --git a/tests/test_mc.sh b/tests/test_mc.sh index dbde75c..23c91d6 100755 --- a/tests/test_mc.sh +++ b/tests/test_mc.sh @@ -22,12 +22,13 @@ source ./tests/test_common.sh source ./tests/commands/delete_bucket_policy.sh source ./tests/commands/get_bucket_policy.sh source ./tests/commands/put_bucket_policy.sh +source ./tests/drivers/create_bucket/create_bucket.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh +source ./tests/drivers/get_bucket_location/get_bucket_location.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh source ./tests/drivers/get_object_tagging/get_object_tagging.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh -source ./tests/util/util_create_bucket.sh source ./tests/util/util_head_bucket.sh export RUN_MC=true diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 0f1efe2..7baa00a 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -41,7 +41,6 @@ source ./tests/drivers/file.sh source ./tests/drivers/xml.sh source ./tests/logger.sh source ./tests/setup.sh -source ./tests/util/util_chunked_upload.sh source ./tests/util/util_delete_object.sh source ./tests/util/util_legal_hold.sh source ./tests/util/util_list_buckets.sh diff --git a/tests/test_rest_chunked.sh b/tests/test_rest_chunked.sh index 214d12b..e15be7e 100755 --- a/tests/test_rest_chunked.sh +++ b/tests/test_rest_chunked.sh @@ -23,7 +23,6 @@ source ./tests/drivers/file.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh source ./tests/drivers/get_object_lock_config/get_object_lock_config_rest.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh -source ./tests/util/util_chunked_upload.sh source ./tests/util/util_file.sh @test "REST - chunked upload, no content length" { diff --git a/tests/test_s3api_bucket.sh b/tests/test_s3api_bucket.sh index 869e147..f65c727 100755 --- a/tests/test_s3api_bucket.sh +++ b/tests/test_s3api_bucket.sh @@ -45,13 +45,14 @@ source ./tests/commands/put_public_access_block.sh source ./tests/commands/select_object_content.sh source ./tests/drivers/file.sh source ./tests/drivers/user.sh +source ./tests/drivers/create_bucket/create_bucket.sh source ./tests/drivers/get_bucket_acl/get_bucket_acl.sh +source ./tests/drivers/get_bucket_location/get_bucket_location.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging.sh source ./tests/drivers/get_bucket_tagging/get_bucket_tagging_rest.sh source ./tests/drivers/head_bucket/head_bucket_rest.sh source ./tests/drivers/head_bucket/head_bucket_s3api.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh -source ./tests/util/util_create_bucket.sh source ./tests/util/util_file.sh source ./tests/util/util_head_bucket.sh source ./tests/util/util_lock_config.sh diff --git a/tests/test_s3api_object.sh b/tests/test_s3api_object.sh index 347d503..b8801c4 100755 --- a/tests/test_s3api_object.sh +++ b/tests/test_s3api_object.sh @@ -18,7 +18,6 @@ load ./bats-support/load load ./bats-assert/load source ./tests/setup.sh -source ./tests/util/util_create_bucket.sh source ./tests/util/util_file.sh source ./tests/util/util_lock_config.sh source ./tests/util/util_object.sh diff --git a/tests/test_s3cmd.sh b/tests/test_s3cmd.sh index 29d5ac3..a6b06b9 100755 --- a/tests/test_s3cmd.sh +++ b/tests/test_s3cmd.sh @@ -23,8 +23,8 @@ source ./tests/test_common_acl.sh source ./tests/commands/delete_bucket_policy.sh source ./tests/commands/get_bucket_policy.sh source ./tests/commands/put_bucket_policy.sh +source ./tests/drivers/get_bucket_location/get_bucket_location.sh source ./tests/drivers/put_bucket_ownership_controls/put_bucket_ownership_controls_rest.sh -source ./tests/util/util_create_bucket.sh source ./tests/util/util_object.sh export RUN_S3CMD=true diff --git a/tests/test_user_common.sh b/tests/test_user_common.sh index e164fbe..7c2e644 100755 --- a/tests/test_user_common.sh +++ b/tests/test_user_common.sh @@ -16,7 +16,6 @@ source ./tests/setup.sh source ./tests/drivers/create_bucket/create_bucket_rest.sh -source ./tests/util/util_create_bucket.sh source ./tests/util/util_list_buckets.sh source ./tests/util/util_object.sh source ./tests/util/util_users.sh diff --git a/tests/util/util_bucket_location.sh b/tests/util/util_bucket_location.sh deleted file mode 100644 index fc11afc..0000000 --- a/tests/util/util_bucket_location.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -get_check_bucket_location_various() { - if [ $# -ne 2 ]; then - log 2 "'get_bucket_location' requires client, bucket" - return 1 - fi - if ! get_bucket_location "$1" "$2"; then - log 2 "error getting bucket location" - return 1 - fi - # shellcheck disable=SC2154 - if [[ $bucket_location != "null" ]] && [[ $bucket_location != "us-east-1" ]]; then - log 2 "wrong location: '$bucket_location'" - return 1 - fi - return 0 -} \ No newline at end of file diff --git a/tests/util/util_chunked_upload.sh b/tests/util/util_chunked_upload.sh deleted file mode 100644 index d759b5b..0000000 --- a/tests/util/util_chunked_upload.sh +++ /dev/null @@ -1,310 +0,0 @@ -#!/usr/bin/env bash - -source ./tests/drivers/rest.sh -source ./tests/drivers/openssl.sh - -attempt_seed_signature_without_content_length() { - if ! check_param_count_v2 "bucket, key, data file" 3 $#; then - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ - AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ - DATA_FILE="$1" \ - BUCKET_NAME="$2" \ - OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" NO_CONTENT_LENGTH="true" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then - log 2 "error creating command: $result" - return 1 - fi - if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 411; then - log 2 "error in sending or checking response code" - return 1 - fi - return 0 -} - -attempt_chunked_upload_with_bad_first_signature() { - if [ $# -ne 3 ]; then - log 2 "'attempt_chunked_upload_with_bad_first_signature' requires data file, bucket name, key" - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ - AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ - DATA_FILE="$1" \ - BUCKET_NAME="$2" \ - OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" FIRST_SIGNATURE="xxxxxxxx" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then - log 2 "error creating command: $result" - return 1 - fi - - if ! result=$(send_via_openssl "$TEST_FILE_FOLDER/command.txt"); then - log 2 "error sending command via openssl" - return 1 - fi - log 5 "result: $result" - echo -n "$result" > "$TEST_FILE_FOLDER/result.txt" - if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error_data.txt"; then - log 2 "error parsing XML data from result" - return 1 - fi - response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')" - if ! check_rest_expected_error "$response_code" "$TEST_FILE_FOLDER/error_data.txt" "403" "SignatureDoesNotMatch" "does not match"; then - log 2 "error checking expected REST error" - return 1 - fi - return 0 -} - -chunked_upload_success() { - if ! check_param_count_v2 "data file, bucket name, key" 3 $#; then - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ - AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ - DATA_FILE="$1" \ - BUCKET_NAME="$2" \ - OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then - log 2 "error creating command: $result" - return 1 - fi - - if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then - log 2 "error sending command via openssl or checking response code" - return 1 - fi - return 0 -} - -attempt_chunked_upload_with_bad_final_signature() { - if [ $# -ne 3 ]; then - log 2 "'attempt_chunked_upload_with_bad_first_signature' requires data file, bucket name, key" - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ - AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ - DATA_FILE="$1" \ - BUCKET_NAME="$2" \ - OBJECT_KEY="$3" \ - CHUNK_SIZE=8192 \ - TEST_MODE=false \ - COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" \ - FINAL_SIGNATURE="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" ./tests/rest_scripts/put_object_openssl_chunked_example.sh 2>&1); then - log 2 "error creating command: $result" - return 1 - fi - if ! result=$(send_via_openssl "$TEST_FILE_FOLDER/command.txt"); then - log 2 "error sending command via openssl" - return 1 - fi - log 5 "response: $result" - echo -n "$result" > "$TEST_FILE_FOLDER/result.txt" - if ! get_xml_data "$TEST_FILE_FOLDER/result.txt" "$TEST_FILE_FOLDER/error_data.txt"; then - log 2 "error parsing XML data from result" - return 1 - fi - log 5 "xml data: $(cat "$TEST_FILE_FOLDER/error_data.txt")" - response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')" - if ! check_rest_expected_error "$response_code" "$TEST_FILE_FOLDER/error_data.txt" "403" "SignatureDoesNotMatch" "does not match"; then - log 2 "error checking expected REST error" - return 1 - fi - return 0 -} - -put_object_chunked_trailer_success() { - if [ $# -ne 4 ]; then - log 2 "'put_object_chunked_trailer_success' requires data file, bucket, key, checksum type" - return 1 - fi - # shellcheck disable=SC2097,SC2098 - if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ - AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ - DATA_FILE="$1" \ - BUCKET_NAME="$2" \ - OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false TRAILER="x-amz-checksum-$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then - log 2 "error creating command: $result" - return 1 - fi - - if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then - log 2 "error sending command via openssl or checking response code" - return 1 - fi - return 0 -} - -put_chunked_upload_trailer_invalid() { - if [ $# -ne 3 ]; then - log 2 "'put_object_chunked_trailer_success' requires data file, bucket, key" - return 1 - fi - # shellcheck disable=SC2097,SC2098 - if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ - AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ - DATA_FILE="$1" \ - BUCKET_NAME="$2" \ - OBJECT_KEY="$3" CHUNK_SIZE=8192 TEST_MODE=false \ - TRAILER="x-amz-checksum-sha10" \ - INVALID_CHECKSUM_TYPE="true" CHECKSUM="abc" \ - TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then - log 2 "error creating command: $result" - return 1 - fi - - if ! result=$(send_via_openssl "$TEST_FILE_FOLDER/command.txt"); then - log 2 "error sending command via openssl" - return 1 - fi - response_code="$(echo "$result" | grep "HTTP" | awk '{print $2}')" - if [ "$response_code" != "400" ]; then - log 2 "expected response '400', was '$response_code'" - return 1 - fi - error_data="$(echo "$result" | grep "" | sed 's/---//g')" - echo -n "$error_data" > "$TEST_FILE_FOLDER/error-data.txt" - if ! check_xml_error_contains "$TEST_FILE_FOLDER/error-data.txt" "InvalidRequest" "The value specified in the x-amz-trailer header is not supported"; then - log 2 "error checking xml error, message" - return 1 - fi - return 0 -} - -chunked_upload_trailer_success() { - if [ $# -ne 1 ]; then - log 2 "'chunked_upload_trailer_success' requires checksum" - return 1 - fi - if ! bucket_name=$(get_bucket_name "$BUCKET_ONE_NAME" 2>&1); then - log 2 "error getting bucket name: $bucket_name" - return 1 - fi - if ! setup_bucket "$bucket_name"; then - log 2 "error setting up bucket" - return 1 - fi - test_file="test-file" - if ! create_test_file "$test_file" 10000; then - log 2 "error creating test file" - return 1 - fi - if ! put_object_chunked_trailer_success "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$1"; then - log 2 "error performing chunked upload w/trailer" - return 1 - fi - if ! download_and_compare_file "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"; then - log 2 "error downloading and comparing file" - return 1 - fi - return 0 -} - -chunked_upload_trailer_invalid_checksum() { - if [ "$#" -ne 1 ]; then - log 2 "'chunked_upload_trailer_invalid_checksum' requires checksum" - return 1 - fi - if ! bucket_name=$(get_bucket_name "$BUCKET_ONE_NAME" 2>&1); then - log 2 "error getting bucket name: $bucket_name" - return 1 - fi - if ! setup_bucket "$bucket_name"; then - log 2 "error setting up bucket" - return 1 - fi - test_file="test-file" - if ! create_test_file "$test_file" 10000; then - log 2 "error creating test file" - return 1 - fi - # shellcheck disable=SC2097,SC2098 - if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ - AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ - DATA_FILE="$TEST_FILE_FOLDER/$test_file" \ - BUCKET_NAME="$bucket_name" \ - OBJECT_KEY="$test_file" CHUNK_SIZE=8192 TEST_MODE=false TRAILER="x-amz-checksum-$1" CHECKSUM="a" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then - log 2 "error creating command: $result" - return 1 - fi - if ! send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/command.txt" "400" "InvalidRequest" "Value for x-amz-checksum-$1 trailing header is invalid."; then - log 2 "error sending openssl and checking response" - return 1 - fi - return 0 -} - -chunked_upload_trailer_incorrect_checksum() { - if [ "$#" -ne 1 ]; then - log 2 "'chunked_upload_trailer_invalid_checksum' requires checksum" - return 1 - fi - if ! bucket_name=$(get_bucket_name "$BUCKET_ONE_NAME" 2>&1); then - log 2 "error getting bucket name: $bucket_name" - return 1 - fi - if ! setup_bucket "$bucket_name"; then - log 2 "error setting up bucket" - return 1 - fi - test_file="test-file" - if ! create_test_file "$test_file" 10000; then - log 2 "error creating test file" - return 1 - fi - if ! checksum=$(calculate_incorrect_checksum "$1" "$TEST_FILE_FOLDER/$test_file"); then - log 2 "error calculating incorrect checksum" - return 1 - fi - # shellcheck disable=SC2097,SC2098 - if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ - AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ - DATA_FILE="$TEST_FILE_FOLDER/$test_file" \ - BUCKET_NAME="$bucket_name" \ - OBJECT_KEY="$test_file" CHUNK_SIZE=8192 TEST_MODE=false TRAILER="x-amz-checksum-$1" CHECKSUM="$checksum" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then - log 2 "error creating command: $result" - return 1 - fi - uppercase_type="$(echo "$1" | tr '[:lower:]' '[:upper:]')" - if ! send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/command.txt" "400" "BadDigest" "The $uppercase_type you specified did not match the calculated checksum."; then - log 2 "error sending openssl and checking response" - return 1 - fi - return 0 -} - -chunked_upload_trailer_different_chunk_size() { - if ! check_param_count_v2 "data file, bucket, key, checksum type" 4 $#; then - return 1 - fi - # shellcheck disable=SC2097,SC2098 - if ! result=$(COMMAND_LOG="$COMMAND_LOG" \ - AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - AWS_ENDPOINT_URL="$AWS_ENDPOINT_URL" \ - DATA_FILE="$1" \ - BUCKET_NAME="$2" \ - OBJECT_KEY="$3" CHUNK_SIZE=16384 TEST_MODE=false TRAILER="x-amz-checksum-$4" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" COMMAND_FILE="$TEST_FILE_FOLDER/command.txt" ./tests/rest_scripts/put_object_openssl_chunked_trailer_example.sh 2>&1); then - log 2 "error creating command: $result" - return 1 - fi - - if ! send_via_openssl_and_check_code "$TEST_FILE_FOLDER/command.txt" 200; then - log 2 "error sending command via openssl or checking response code" - return 1 - fi - return 0 -} diff --git a/tests/util/util_create_bucket.sh b/tests/util/util_create_bucket.sh deleted file mode 100644 index 3b0e480..0000000 --- a/tests/util/util_create_bucket.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2024 Versity Software -# This file is licensed under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source ./tests/util/util_mc.sh -source ./tests/logger.sh - -create_bucket_invalid_name() { - if [ $# -ne 1 ]; then - log 2 "create bucket w/invalid name missing command type" - return 1 - fi - local exit_code=0 - if [[ $1 == "aws" ]] || [[ $1 == 's3' ]]; then - bucket_create_error=$(aws --no-verify-ssl s3 mb "s3://" 2>&1) || exit_code=$? - elif [[ $1 == 's3api' ]]; then - bucket_create_error=$(aws --no-verify-ssl s3api create-bucket --bucket "s3://" 2>&1) || exit_code=$? - elif [[ $1 == 's3cmd' ]]; then - bucket_create_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb "s3://" 2>&1) || exit_code=$? - elif [[ $1 == 'mc' ]]; then - bucket_create_error=$(mc --insecure mb "$MC_ALIAS/." 2>&1) || exit_code=$? - else - log 2 "invalid command type $1" - return 1 - fi - if [ $exit_code -eq 0 ]; then - log 2 "error: bucket should have not been created but was" - return 1 - fi - echo "$bucket_create_error" -} - -create_and_check_bucket_invalid_name() { - if [ $# -ne 1 ]; then - log 2 "'create_and_check_bucket_invalid_name' requires client" - return 1 - fi - if ! create_bucket_invalid_name "$1"; then - log 2 "error creating bucket with invalid name" - return 1 - fi - - # shellcheck disable=SC2154 - if [[ "$bucket_create_error" != *"Invalid bucket name "* ]] && [[ "$bucket_create_error" != *"Bucket name cannot"* ]]; then - log 2 "unexpected error: $bucket_create_error" - return 1 - fi - return 0 -} - -create_bucket_rest() { - if ! check_param_count "create_bucket_rest" "bucket name" 1 $#; then - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/create_bucket.sh 2>&1); then - log 2 "error creating bucket: $result" - return 1 - fi - if [ "$result" != "200" ]; then - bucket_create_error="$(cat "$TEST_FILE_FOLDER/result.txt")" - log 2 "expected '200', was '$result' ($bucket_create_error)" - return 1 - fi - return 0 -} diff --git a/tests/util/util_object.sh b/tests/util/util_object.sh index 5c6ab19..34b9681 100644 --- a/tests/util/util_object.sh +++ b/tests/util/util_object.sh @@ -14,7 +14,6 @@ # specific language governing permissions and limitations # under the License. -source ./tests/util/util_create_bucket.sh source ./tests/util/util_mc.sh source ./tests/util/util_multipart.sh source ./tests/util/util_versioning.sh