test: REST HeadObject, x-amz-checksum-sha256 test

This commit is contained in:
Luke McCrone
2025-02-13 22:53:53 -03:00
parent f42c20297b
commit 593b53d829
21 changed files with 267 additions and 314 deletions

View File

@@ -22,27 +22,26 @@ source ./tests/rest_scripts/rest.sh
bucket_name="$BUCKET_NAME"
# shellcheck disable=SC2154
key="$OBJECT_KEY"
# shellcheck disable=SC2154
checksum_mode="${CHECKSUM_MODE:=false}"
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
#x-amz-object-attributes:ETag
canonical_request="GET
/$bucket_name/$key
canonical_request_data+=("GET" "/$bucket_name/$key" "" "host:$host")
if [ "$checksum_mode" == "true" ]; then
canonical_request_data+=("x-amz-checksum-mode:ENABLED")
fi
canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
host:$host
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD"
build_canonical_request "${canonical_request_data[@]}"
# shellcheck disable=SC2119
create_canonical_hash_sts_and_signature
curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key"
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
-H "\"x-amz-date: $current_date_time\""
-o "$OUTPUT_FILE")
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
curl_command+=("${header_fields[@]}")
curl_command+=(-o "$OUTPUT_FILE")
# shellcheck disable=SC2154
eval "${curl_command[*]}" 2>&1

View File

@@ -36,13 +36,21 @@ x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD"
canonical_request_data=("HEAD" "/$bucket_name/$key" "" "host:$host")
if [ "$CHECKSUM" == "true" ]; then
canonical_request_data+=("x-amz-checksum-mode:ENABLED")
fi
canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
if ! build_canonical_request "${canonical_request_data[@]}"; then
log_rest 2 "error building request"
exit 1
fi
# shellcheck disable=SC2119
create_canonical_hash_sts_and_signature
curl_command+=(curl -ksI -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key"
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
-H "\"x-amz-date: $current_date_time\""
-o "$OUTPUT_FILE")
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
curl_command+=("${header_fields[@]}")
curl_command+=(-o "$OUTPUT_FILE")
# shellcheck disable=SC2154
eval "${curl_command[*]}" 2>&1

View File

@@ -29,45 +29,21 @@ checksum="$CHECKSUM"
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
payload_hash="$(sha256sum "$data_file" | awk '{print $1}')"
checksum_hash="$(echo -n "$payload_hash" | xxd -r -p | base64)"
if [ "$CHECKSUM" == "true" ]; then
canonical_request="PUT
/$bucket_name/$key
host:$host
x-amz-checksum-sha256:$checksum_hash
x-amz-content-sha256:$payload_hash
x-amz-date:$current_date_time
host;x-amz-checksum-sha256;x-amz-content-sha256;x-amz-date
$payload_hash"
else
canonical_request="PUT
/$bucket_name/$key
host:$host
x-amz-content-sha256:$payload_hash
x-amz-date:$current_date_time
host;x-amz-content-sha256;x-amz-date
$payload_hash"
cr_data=("PUT" "/$bucket_name/$key" "" "host:$host")
if [ "$checksum" == "true" ]; then
checksum_hash="$(echo -n "$payload_hash" | xxd -r -p | base64)"
cr_data+=("x-amz-checksum-sha256:$checksum_hash")
fi
cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
build_canonical_request "${cr_data[@]}"
# shellcheck disable=SC2119
create_canonical_hash_sts_and_signature
curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name/$key")
if [ "$CHECKSUM" == "true" ]; then
curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-checksum-sha256,Signature=$signature\"")
else
curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"")
fi
curl_command+=(-H "\"x-amz-content-sha256: $payload_hash\""
-H "\"x-amz-date: $current_date_time\"")
if [ "$checksum" == "true" ]; then
curl_command+=(-H "\"x-amz-checksum-sha256: $checksum_hash\"")
fi
curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
curl_command+=("${header_fields[@]}")
curl_command+=(-T "$data_file" -o "$OUTPUT_FILE")
# shellcheck disable=SC2154
eval "${curl_command[*]}" 2>&1

View File

@@ -100,3 +100,32 @@ log_rest() {
echo "$2"
fi
}
build_canonical_request() {
if [ $# -lt 0 ]; then
log_rest 2 "'build_canonical_request' requires parameters"
return 1
fi
canonical_request=""
param_list=""
local payload=""
header_fields=()
for line in "$@"; do
canonical_request+="$line
"
if [[ "$line" == *":"* ]]; then
local key="${line%%:*}"
local value="${line#*:}"
if [ "$key" == "x-amz-content-sha256" ]; then
payload="$value"
fi
if [[ "$value" != "" ]]; then
param_list=$(add_parameter "$param_list" "$key" ";")
header_fields+=(-H "\"$key: $value\"")
fi
fi
done
canonical_request+="
$param_list
$payload"
}

View File

@@ -147,6 +147,8 @@ run_suite() {
exit_code=1
elif ! "$HOME"/bin/bats ./tests/test_rest_chunked.sh; then
exit_code=1
elif ! "$HOME"/bin/bats ./tests/test_rest_checksum.sh; then
exit_code=1
fi
;;
s3api-user)

View File

@@ -29,6 +29,7 @@ setup() {
log 2 "error creating log file: $error"
exit 1
fi
export TEST_LOG_FILE
fi
log 4 "Running test $BATS_TEST_NAME"

View File

@@ -22,6 +22,7 @@ source ./tests/util/util_list_buckets.sh
source ./tests/util/util_object.sh
source ./tests/util/util_policy.sh
source ./tests/util/util_presigned_url.sh
source ./tests/util/util_setup.sh
source ./tests/commands/copy_object.sh
source ./tests/commands/delete_bucket_tagging.sh
source ./tests/commands/delete_object_tagging.sh
@@ -43,10 +44,7 @@ test_common_multipart_upload() {
assert [ $# -eq 1 ]
bucket_file="largefile"
run create_large_file "$bucket_file"
assert_success
run setup_bucket "$1" "$BUCKET_ONE_NAME"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
if [ "$1" == 's3' ]; then
@@ -181,10 +179,7 @@ test_common_put_get_object() {
fi
local object_name="test-object"
run create_test_files "$object_name"
assert_success
run setup_bucket "$1" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$object_name"
assert_success
if [[ $1 == 's3' ]]; then
@@ -224,11 +219,7 @@ test_common_list_objects() {
object_one="test-file-one"
object_two="test-file-two"
run create_test_files $object_one $object_two
assert_success
run setup_bucket "$1" "$BUCKET_ONE_NAME"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two"
assert_success
run put_object "$1" "$TEST_FILE_FOLDER"/$object_one "$BUCKET_ONE_NAME" "$object_one"
@@ -273,10 +264,7 @@ test_common_set_get_object_tags() {
local key="test_key"
local value="test_value"
run create_test_files "$bucket_file"
assert_success
run setup_bucket "$1" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
run put_object "$1" "$TEST_FILE_FOLDER"/"$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
@@ -342,10 +330,7 @@ test_common_delete_object_tagging() {
tag_key="key"
tag_value="value"
run create_test_files "$bucket_file"
assert_success
run setup_bucket "$1" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
run put_object "$1" "$TEST_FILE_FOLDER"/"$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"

View File

@@ -46,6 +46,7 @@ source ./tests/util/util_ownership.sh
source ./tests/util/util_policy.sh
source ./tests/util/util_public_access_block.sh
source ./tests/util/util_rest.sh
source ./tests/util/util_setup.sh
source ./tests/util/util_tags.sh
source ./tests/util/util_time.sh
source ./tests/util/util_versioning.sh
@@ -54,11 +55,8 @@ source ./tests/util/util_xml.sh
export RUN_USERS=true
@test "test_rest_list_objects" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
run create_test_files "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -77,11 +75,8 @@ export RUN_USERS=true
}
@test "test_rest_delete_object" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
run create_test_files "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -105,10 +100,7 @@ export RUN_USERS=true
test_key="TestKey"
test_value="TestValue"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run create_test_files "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -199,15 +191,7 @@ export RUN_USERS=true
@test "test_rest_versioning" {
test_file="test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
if [ "$DIRECT" == "true" ]; then
sleep 10
fi
run create_test_file "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -231,11 +215,7 @@ export RUN_USERS=true
@test "versioning - add version, then delete and check for marker" {
test_file="test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run create_test_file "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -253,11 +233,7 @@ export RUN_USERS=true
@test "versioning - retrieve after delete" {
test_file="test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run create_test_file "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -275,11 +251,7 @@ export RUN_USERS=true
@test "REST - legal hold, get without config" {
test_file="test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run create_test_file "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -301,16 +273,12 @@ export RUN_USERS=true
@test "REST - multipart upload create, list parts" {
test_file="test_file"
run create_large_file "$test_file"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run split_file "$TEST_FILE_FOLDER/$test_file" 4
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run upload_check_parts "$BUCKET_ONE_NAME" "$test_file" \
"$TEST_FILE_FOLDER/$test_file-0" "$TEST_FILE_FOLDER/$test_file-1" "$TEST_FILE_FOLDER/$test_file-2" "$TEST_FILE_FOLDER/$test_file-3"
assert_success
@@ -327,11 +295,7 @@ export RUN_USERS=true
skip "https://github.com/versity/versitygw/issues/1000"
fi
test_file="test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run create_large_file "$test_file"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
# shellcheck disable=SC2034
@@ -349,11 +313,7 @@ export RUN_USERS=true
skip "https://github.com/versity/versitygw/issues/1001"
fi
test_file="test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run create_test_file "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -368,11 +328,7 @@ export RUN_USERS=true
skip "https://github.com/versity/versitygw/issues/1006"
fi
test_file="test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run create_test_file "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run add_and_check_checksum "$TEST_FILE_FOLDER/$test_file" "$test_file"
@@ -446,13 +402,10 @@ export RUN_USERS=true
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/993"
fi
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
test_file_two="test_file_2"
test_file_three="test_file_3"
run create_test_files "$test_file" "$test_file_two" "$test_file_three"
run setup_bucket_and_files "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" "$test_file_three"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -477,12 +430,9 @@ export RUN_USERS=true
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/999"
fi
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
test_file_two="test_file_2"
run create_test_files "$test_file" "$test_file_two"
run setup_bucket "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_two"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -499,11 +449,8 @@ export RUN_USERS=true
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1008"
fi
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
run create_large_file "$test_file"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run create_upload_finish_wrong_etag "$BUCKET_ONE_NAME" "$test_file"
@@ -511,11 +458,8 @@ export RUN_USERS=true
}
@test "REST - upload part copy" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
run create_large_file "$test_file"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run create_upload_part_copy_rest "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
@@ -529,11 +473,8 @@ export RUN_USERS=true
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1018"
fi
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
run create_test_file "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -567,12 +508,9 @@ export RUN_USERS=true
}
@test "REST - delete objects command" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file_one="test_file"
test_file_two="test_file_two"
run create_test_files "$test_file_one" "$test_file_two"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$test_file_one" "$test_file_two"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_one" "$BUCKET_ONE_NAME" "$test_file_one"
@@ -595,4 +533,4 @@ export RUN_USERS=true
run verify_object_not_found "$BUCKET_ONE_NAME" "$test_file_two"
assert_success
}
}

View File

@@ -22,6 +22,7 @@ source ./tests/logger.sh
source ./tests/setup.sh
source ./tests/util/util_acl.sh
source ./tests/util/util_object.sh
source ./tests/util/util_setup.sh
export RUN_USERS=true
@@ -34,11 +35,8 @@ export RUN_USERS=true
}
@test "REST - put ACL" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
run create_test_files "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred"
@@ -72,11 +70,8 @@ export RUN_USERS=true
}
@test "REST - put public-read canned acl" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
run create_test_files "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred"
@@ -135,11 +130,8 @@ export RUN_USERS=true
}
@test "REST - put public-read-write canned acl" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test_file"
run create_test_files "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred"

37
tests/test_rest_checksum.sh Executable file
View File

@@ -0,0 +1,37 @@
#!/usr/bin/env bats
# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
load ./bats-support/load
load ./bats-assert/load
source ./tests/setup.sh
source ./tests/util/util_head_object.sh
source ./tests/util/util_setup.sh
@test "REST - HeadObject returns x-amz-checksum-sha256" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1070"
fi
test_file="test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object_rest_sha256_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
assert_success
run check_checksum_rest "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
assert_success
}

View File

@@ -23,16 +23,13 @@ source ./tests/util/util_bucket.sh
source ./tests/util/util_chunked_upload.sh
source ./tests/util/util_file.sh
source ./tests/util/util_head_object.sh
source ./tests/util/util_setup.sh
@test "REST - chunked upload, no content length" {
if [ "$DIRECT" != "true" ]; then
skip "https://github.com/versity/versitygw/issues/1043"
fi
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
test_file="test-file"
run create_test_files "$test_file"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run attempt_seed_signature_without_content_length "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"

View File

@@ -23,6 +23,7 @@ source ./tests/util/util_file.sh
source ./tests/util/util_multipart.sh
source ./tests/util/util_multipart_abort.sh
source ./tests/util/util_multipart_before_completion.sh
source ./tests/util/util_setup.sh
source ./tests/util/util_tags.sh
source ./tests/commands/get_object.sh
source ./tests/commands/put_object.sh
@@ -31,9 +32,6 @@ source ./tests/commands/list_multipart_uploads.sh
# abort-multipart-upload
@test "test_abort_multipart_upload" {
local bucket_file="bucket-file"
run create_test_file "$bucket_file"
assert_success
# shellcheck disable=SC2154
run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
assert_success
@@ -51,9 +49,6 @@ source ./tests/commands/list_multipart_uploads.sh
# complete-multipart-upload
@test "test_complete_multipart_upload" {
local bucket_file="bucket-file"
run create_test_files "$bucket_file"
assert_success
run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=20M count=1
assert_success
@@ -111,10 +106,6 @@ source ./tests/commands/list_multipart_uploads.sh
@test "test-multipart-upload-from-bucket" {
local bucket_file="bucket-file"
run create_test_file "$bucket_file"
assert_success
run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=20M count=1
assert_success
@@ -133,10 +124,7 @@ source ./tests/commands/list_multipart_uploads.sh
@test "test_multipart_upload_from_bucket_range_too_large" {
local bucket_file="bucket-file"
run create_large_file "$bucket_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file"
@@ -145,10 +133,7 @@ source ./tests/commands/list_multipart_uploads.sh
@test "test_multipart_upload_from_bucket_range_valid" {
local bucket_file="bucket-file"
run create_large_file "$bucket_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file"
@@ -158,9 +143,6 @@ source ./tests/commands/list_multipart_uploads.sh
# test multi-part upload list parts command
@test "test-multipart-upload-list-parts" {
local bucket_file="bucket-file"
run create_test_file "$bucket_file" 0
assert_success
run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
assert_success
@@ -176,18 +158,14 @@ source ./tests/commands/list_multipart_uploads.sh
# test listing of active uploads
@test "test-multipart-upload-list-uploads" {
local bucket_file_one="bucket-file-one"
local bucket_file_two="bucket-file-two"
if [[ $RECREATE_BUCKETS == false ]]; then
run abort_all_multipart_uploads "$BUCKET_ONE_NAME"
assert_success
fi
run create_test_files "$bucket_file_one" "$bucket_file_two"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
local bucket_file_one="bucket-file-one"
local bucket_file_two="bucket-file-two"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two"
assert_success
run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two"

View File

@@ -22,6 +22,7 @@ source ./tests/util/util_create_bucket.sh
source ./tests/util/util_file.sh
source ./tests/util/util_lock_config.sh
source ./tests/util/util_object.sh
source ./tests/util/util_setup.sh
source ./tests/util/util_tags.sh
source ./tests/util/util_users.sh
source ./tests/test_s3api_root_inner.sh
@@ -111,10 +112,7 @@ export RUN_USERS=true
run create_test_folder "$folder_name"
assert_success
run create_test_file "$folder_name"/"$object_name"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$folder_name/$object_name"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$folder_name/$object_name" "$BUCKET_ONE_NAME" "$folder_name/$object_name"
@@ -167,14 +165,11 @@ export RUN_USERS=true
}
@test "test_put_object_metadata" {
object_one="object-one"
test_key="x-test-data"
test_value="test-value"
run create_test_files "$object_one"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
object_one="object-one"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$object_one"
assert_success
object="$TEST_FILE_FOLDER"/"$object_one"
@@ -242,10 +237,7 @@ export RUN_USERS=true
fi
test_file="a"
run create_test_file "$test_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file/"

View File

@@ -151,9 +151,8 @@ test_s3api_policy_put_acl() {
run create_test_file "$policy_file" 0
assert_success
run create_large_file "$test_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred"

View File

@@ -15,15 +15,13 @@
# under the License.
source ./tests/util/util_multipart_abort.sh
source ./tests/util/util_setup.sh
test_s3api_policy_abort_multipart_upload() {
policy_file="policy_file"
test_file="test_file"
run create_large_file "$test_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run setup_user_versitygw_or_direct "$USERNAME_ONE" "$PASSWORD_ONE" "user" "$BUCKET_ONE_NAME"
@@ -66,7 +64,7 @@ test_s3api_policy_list_multipart_uploads() {
run create_test_file "$policy_file"
assert_success
run create_large_file "$test_file"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
effect="Allow"
@@ -79,9 +77,6 @@ test_s3api_policy_list_multipart_uploads() {
username=${lines[0]}
password=${lines[1]}
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
assert_success
@@ -106,10 +101,7 @@ test_s3api_policy_list_upload_parts() {
run create_test_files "$policy_file"
assert_success "error creating test files"
run create_large_file "$test_file"
assert_success "error creating large file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success "error setting up bucket"
run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user"

View File

@@ -15,6 +15,7 @@
# under the License.
source ./tests/util/util_delete_object.sh
source ./tests/util/util_setup.sh
test_s3api_policy_allow_deny() {
policy_file="policy_file"
@@ -22,13 +23,13 @@ test_s3api_policy_allow_deny() {
username=$USERNAME_ONE
password=$PASSWORD_ONE
run create_test_files "$policy_file" "$test_file"
run create_test_files "$policy_file"
assert_success
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" \
@@ -53,7 +54,7 @@ test_s3api_policy_delete() {
username=$USERNAME_ONE
password=$PASSWORD_ONE
run create_test_files "$test_file_one" "$test_file_two" "$policy_file"
run create_test_files "$policy_file"
assert_success
effect="Allow"
@@ -64,7 +65,7 @@ test_s3api_policy_delete() {
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$test_file_one" "$test_file_two"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
@@ -94,13 +95,13 @@ test_s3api_policy_deny() {
username=$USERNAME_ONE
password=$PASSWORD_ONE
run create_test_files "$test_file_one" "$test_file_two" "$policy_file"
run create_test_files "$policy_file"
assert_success
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$test_file_one" "$test_file_two"
assert_success
run setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" \
@@ -131,9 +132,6 @@ test_s3api_policy_get_object_file_wildcard() {
username=$USERNAME_ONE
password=$PASSWORD_ONE
run create_test_files "$policy_file" "$policy_file_two" "$policy_file_three"
assert_success
effect="Allow"
principal="$username"
action="s3:GetObject"
@@ -142,7 +140,7 @@ test_s3api_policy_get_object_file_wildcard() {
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$policy_file_two" "$policy_file_three"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
@@ -177,9 +175,6 @@ test_s3api_policy_get_object_folder_wildcard() {
run create_test_folder "$test_folder"
assert_success
run create_test_files "$test_folder/$test_file" "$policy_file"
assert_success
effect="Allow"
principal="$username"
action="s3:GetObject"
@@ -188,7 +183,7 @@ test_s3api_policy_get_object_folder_wildcard() {
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_folder/$test_file"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
@@ -211,7 +206,10 @@ test_s3api_policy_get_object_specific_file() {
username=$USERNAME_ONE
password=$PASSWORD_ONE
run create_test_files "$policy_file" "$test_file" "$test_file_two"
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$test_file" "$test_file_two"
assert_success
effect="Allow"
@@ -219,12 +217,6 @@ test_s3api_policy_get_object_specific_file() {
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/test_file"
run setup_user "$username" "$password" "user"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
assert_success
@@ -245,21 +237,11 @@ test_s3api_policy_get_object_specific_file() {
}
test_s3api_policy_get_object_with_user() {
policy_file="policy_file"
username=$USERNAME_ONE
password=$PASSWORD_ONE
test_file="test_file"
log 5 "username: $USERNAME_ONE, password: $PASSWORD_ONE"
run create_test_files "$test_file" "$policy_file"
assert_success
effect="Allow"
principal="$username"
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -271,6 +253,12 @@ test_s3api_policy_get_object_with_user() {
run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
assert_success
policy_file="policy_file"
effect="Allow"
principal="$username"
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
assert_success
@@ -283,10 +271,6 @@ test_s3api_policy_get_object_with_user() {
test_s3api_policy_invalid_action() {
policy_file="policy_file"
run create_test_file "$policy_file"
assert_success
effect="Allow"
principal="*"
action="s3:GetObjectt"
@@ -314,16 +298,13 @@ test_s3api_policy_put_wildcard() {
run create_test_folder "$test_folder"
assert_success
run create_test_files "$test_folder/$test_file" "$policy_file"
assert_success
run setup_user_versitygw_or_direct "$USERNAME_ONE" "$PASSWORD_ONE" "user" "$BUCKET_ONE_NAME"
assert_success
# shellcheck disable=SC2154
username=${lines[0]}
password=${lines[1]}
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_folder/$test_file"
assert_success
run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "Allow" "$username" "s3:PutObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*"
@@ -351,9 +332,7 @@ test_s3api_policy_two_principals() {
policy_file="policy_file"
test_file="test_file"
run create_test_files "$test_file" "$policy_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run setup_user "$USERNAME_ONE" "$PASSWORD_ONE" "user"
assert_success

View File

@@ -23,15 +23,12 @@ source ./tests/util/util_get_object_retention.sh
source ./tests/util/util_head_object.sh
source ./tests/util/util_legal_hold.sh
source ./tests/util/util_list_objects.sh
source ./tests/util/util_setup.sh
test_delete_objects_s3api_root() {
local object_one="test-file-one"
local object_two="test-file-two"
run create_test_files "$object_one" "$object_two"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$BUCKET_ONE_NAME" "$object_one"
@@ -52,9 +49,6 @@ test_delete_objects_s3api_root() {
test_get_object_full_range_s3api_root() {
bucket_file="bucket_file"
run create_test_files "$bucket_file" 0
assert_success
echo -n "0123456789" > "$TEST_FILE_FOLDER/$bucket_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
@@ -71,10 +65,7 @@ test_get_object_full_range_s3api_root() {
test_get_object_invalid_range_s3api_root() {
bucket_file="bucket_file"
run create_test_files "$bucket_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
@@ -105,10 +96,7 @@ test_put_object_s3api_root() {
test_get_object_attributes_s3api_root() {
bucket_file="bucket_file"
run create_test_file "$bucket_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$bucket_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file"
@@ -252,11 +240,7 @@ legal_hold_retention_setup() {
test_s3api_list_objects_v1_s3api_root() {
local object_one="test-file-one"
local object_two="test-file-two"
run create_test_files "$object_one" "$object_two"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$BUCKET_ONE_NAME" "$object_one"
@@ -272,11 +256,7 @@ test_s3api_list_objects_v1_s3api_root() {
test_s3api_list_objects_v2_s3api_root() {
local object_one="test-file-one"
local object_two="test-file-two"
run create_test_files "$object_one" "$object_two"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_files "$BUCKET_ONE_NAME" "$object_one" "$object_two"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER"/"$object_one" "$BUCKET_ONE_NAME" "$object_one"

View File

@@ -18,6 +18,7 @@ load ./bats-support/load
load ./bats-assert/load
source ./tests/test_user_common.sh
source ./tests/util/util_setup.sh
source ./tests/util/util_users.sh
source ./tests/commands/get_object.sh
source ./tests/commands/put_object.sh
@@ -54,10 +55,7 @@ export RUN_USERS=true
username=${lines[0]}
password=${lines[1]}
run create_test_file "$test_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
@@ -81,10 +79,7 @@ export RUN_USERS=true
username=${lines[0]}
password=${lines[1]}
run create_test_file "$test_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
@@ -108,10 +103,7 @@ export RUN_USERS=true
username=${lines[0]}
password=${lines[1]}
run create_test_file "$test_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
@@ -135,10 +127,7 @@ export RUN_USERS=true
username=${lines[0]}
password=${lines[1]}
run create_test_file "$test_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run put_object_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password"
@@ -166,10 +155,7 @@ export RUN_USERS=true
username=${lines[0]}
password=${lines[1]}
run create_large_file "$test_file"
assert_success
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file"
assert_success
run create_multipart_upload_with_user "$BUCKET_ONE_NAME" "dummy" "$username" "$password"

View File

@@ -120,3 +120,24 @@ verify_object_exists() {
fi
return 0
}
check_checksum_rest() {
if [ $# -ne 3 ]; then
log 2 "'check_checksum_rest' requires bucket, file, local file"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" CHECKSUM="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/head_object.sh 2>&1); then
log 2 "error: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected response code '200', was '$result'"
return 1
fi
head_checksum=$(grep "x-amz-checksum-sha256" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//')
file_checksum="$(sha256sum "$3" | awk '{print $1}' | xxd -r -p | base64)"
if [ "$file_checksum" != "$head_checksum" ]; then
log 2 "'checksum mismatch (head '$head_checksum', local '$file_checksum')"
return 1
fi
}

View File

@@ -295,4 +295,21 @@ list_and_check_directory_obj() {
fi
fi
return 0
}
}
put_object_rest_sha256_checksum() {
if [ $# -ne 3 ]; then
log 2 "'put_object_rest_sha256_checksum' requires data file, bucket name, key"
return 1
fi
if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM="true" ./tests/rest_scripts/put_object.sh 2>&1); then
log 2 "error: $result"
return 1
fi
if [ "$result" != "200" ]; then
log 2 "expected response code of '200', was '$result'"
return 1
fi
log 5 "result: $(cat "$TEST_FILE_FOLDER/result.txt")"
return 0
}

45
tests/util/util_setup.sh Normal file
View File

@@ -0,0 +1,45 @@
#!/usr/bin/env bash
setup_bucket_and_file() {
if [ $# -ne 2 ]; then
log 2 "'setup_bucket_and_file' requires bucket name, file name"
return 1
fi
if ! setup_bucket_and_files "$1" "$2"; then
log 2 "error setting up bucket and file"
return 1
fi
return 0
}
setup_bucket_and_files() {
if [ $# -lt 2 ]; then
log 2 "'setup_bucket_and_files' requires bucket name, file names"
return 1
fi
if ! setup_bucket "s3api" "$1"; then
log 2 "error setting up bucket"
return 1
fi
if ! create_test_files "${@:2}"; then
log 2 "error creating test files"
return 1
fi
return 0
}
setup_bucket_and_large_file() {
if [ $# -ne 2 ]; then
log 2 "'setup_bucket_and_large_file' requires bucket name, file name"
return 1
fi
if ! setup_bucket "s3api" "$1"; then
log 2 "error setting up bucket"
return 1
fi
if ! create_large_file "$2"; then
log 2 "error creating large file"
return 1
fi
return 0
}