diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 38f6563..fdf7439 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -21,9 +21,9 @@ jobs: RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" - - set: "REST, posix, non-static, base|acl|multipart, folder IAM" + - set: "REST, posix, non-static, base|acl|multipart|put-object, folder IAM" IAM_TYPE: folder - RUN_SET: "rest-base,rest-acl,rest-multipart" + RUN_SET: "rest-base,rest-acl,rest-multipart,rest-put-object" RECREATE_BUCKETS: "true" DELETE_BUCKETS_AFTER_TEST: "true" BACKEND: "posix" diff --git a/tests/commands/head_object.sh b/tests/commands/head_object.sh index 2a79742..ea44da4 100644 --- a/tests/commands/head_object.sh +++ b/tests/commands/head_object.sh @@ -41,4 +41,40 @@ head_object() { fi fi return 0 -} \ No newline at end of file +} + +head_object_rest_expect_success() { + if ! check_param_count_v2 "bucket, object, env vars" 4 $#; then + return 1 + fi + env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2 $3" + if ! send_rest_command_expect_success "$env_vars" "./tests/rest_scripts/head_object.sh" "200"; then + log 2 "error sending REST command and checking error" + return 1 + fi + return 0 +} + +head_object_rest_expect_success_callback() { + if ! check_param_count_v2 "bucket, object, env vars, callback" 4 $#; then + return 1 + fi + env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2 $3" + if ! send_rest_command_expect_success_callback "$env_vars" "./tests/rest_scripts/head_object.sh" "200" "$4"; then + log 2 "error sending REST command and checking error" + return 1 + fi + return 0 +} + +head_object_rest_expect_error() { + if ! check_param_count_v2 "bucket, object, env vars, response code, error code" 5 $#; then + return 1 + fi + env_vars="BUCKET_NAME=$1 OBJECT_KEY=$2 $3" + if ! send_rest_command_expect_header_error "$env_vars" "./tests/rest_scripts/head_object.sh" "$4" "$5"; then + log 2 "error sending HeadObject REST command and checking error" + return 1 + fi + return 0 +} diff --git a/tests/drivers/head_object/head_object_rest.sh b/tests/drivers/head_object/head_object_rest.sh new file mode 100644 index 0000000..9a29609 --- /dev/null +++ b/tests/drivers/head_object/head_object_rest.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +get_etag_rest() { + if ! check_param_count_v2 "bucket name, object key" 2 $#; then + return 1 + fi + if ! head_object_rest_expect_success_callback "$1" "$2" "" "parse_etag"; then + log 2 "error calling HeadObject command" + return 1 + fi + return 0 +} + +parse_etag() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + log 5 "head object data: $(cat "$1")" + etag_value=$(grep "E[Tt]ag:" "$1" | sed -n 's/E[Tt]ag: "\([^"]*\)"/\1/p' | tr -d '\r') + echo "$etag_value" +} + +verify_object_not_found() { + if ! check_param_count_v2 "bucket name, key" 2 $#; then + return 1 + fi + if ! head_object_rest_expect_error "$1" "$2" "" "404" "Not Found"; then + return 1 + fi + return 0 +} + +verify_object_exists() { + if ! check_param_count_v2 "bucket name, key" 2 $#; then + return 1 + fi + if ! head_object_rest_expect_success "$1" "$2" "" "200"; then + log 2 "error sending HeadObject command and verifying existence" + return 1 + fi + return 0 +} + +check_checksum_rest() { + if ! check_param_count_v2 "bucket, file, expected checksum, header key" 4 $#; then + return 1 + fi + header_key="$4" + if ! head_object_rest_expect_success_callback "$1" "$2" "CHECKSUM=true" "parse_head_checksum"; then + log 2 "error calling HeadObject command" + return 1 + fi + if [ "$3" != "$head_checksum" ]; then + log 2 "'checksum mismatch (head '$head_checksum', local '$3')" + return 1 + fi + return 0 +} + +parse_head_checksum() { + if ! check_param_count_v2 "file" 1 $#; then + return 1 + fi + head_checksum=$(grep -i "$header_key" "$1" | awk '{print $2}' | sed 's/\r$//') +} + +verify_checksum_doesnt_exist() { + if ! check_param_count_v2 "file" 1 $#; then + return 1 + fi + head_checksum=$(grep -i "$header_key" "$1" | awk '{print $2}' | sed 's/\r$//') + if [ "$head_checksum" != "" ]; then + log 2 "head checksum shouldn't be returned, is $head_checksum" + return 1 + fi +} + +parse_content_length() { + if ! check_param_count_v2 "file" 1 $#; then + return 1 + fi + content_length=$(grep "Content-Length:" "$1" | awk '{print $2}' | tr -d '\r') + return 0 +} + +check_checksum_rest_sha256() { + if ! check_param_count_v2 "bucket, file, local file" 3 $#; then + return 1 + fi + file_checksum="$(sha256sum "$3" | awk '{print $1}' | xxd -r -p | base64)" + if ! check_checksum_rest "$1" "$2" "$file_checksum" "x-amz-checksum-sha256"; then + log 2 "error checking sha256 checksum" + return 1 + fi + return 0 +} + +check_checksum_rest_crc32() { + if ! check_param_count_v2 "bucket, file, local file" 3 $#; then + return 1 + fi + file_checksum="$(gzip -c -1 "$3" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)" + if ! check_checksum_rest "$1" "$2" "$file_checksum" "x-amz-checksum-crc32"; then + log 2 "error checking crc32 checksum" + return 1 + fi + return 0 +} + +head_object_without_and_with_checksum() { + if ! check_param_count_v2 "bucket, file" 2 $#; then + return 1 + fi + header_key="x-amz-checksum-sha256" + if ! head_object_rest_expect_success_callback "$1" "$2" "" "verify_checksum_doesnt_exist"; then + log 2 "error verifying HeadObject checksum doesn't exist" + return 1 + fi + if ! head_object_rest_expect_success_callback "$1" "$2" "CHECKSUM=true" "parse_head_checksum"; then + log 2 "error verifying checksum exists" + return 1 + fi + return 0 +} + +check_default_checksum() { + if ! check_param_count_v2 "bucket, file, local file" 3 $#; then + return 1 + fi + header_key="x-amz-checksum-crc64nvme" + if ! head_object_rest_expect_success_callback "$1" "$2" "CHECKSUM=true" "parse_head_checksum"; then + log 2 "error verifying HeadObject checksum doesn't exist" + return 1 + fi + log 5 "checksum: $head_checksum" + default_checksum="$head_checksum" + if ! head_object_rest_expect_success_callback "$1" "$2" "CHECKSUM_TYPE=crc64nvme CHECKSUM=true" "parse_head_checksum"; then + log 2 "error verifying HeadObject checksum doesn't exist" + return 1 + fi + if [ "$head_checksum" != "$default_checksum" ]; then + log 2 "checksum mismatch (crc64nvme: '$head_checksum', default: '$default_checksum')" + return 1 + fi + return 0 +} + +get_object_size_with_user() { + if ! check_param_count_v2 "username, password, bucket, key" 4 $#; then + return 1 + fi + if ! head_object_rest_expect_success_callback "$3" "$4" "AWS_ACCESS_KEY_ID=$1 AWS_SECRET_ACCESS_KEY=$2" "parse_content_length"; then + log 2 "error getting object size" + return 1 + fi + log 5 "file size: $content_length" + echo "$content_length" + return 0 +} + +check_metadata_key_case() { + if ! check_param_count_v2 "bucket name, test file, expected mixed-case key, expected value" 4 $#; then + return 1 + fi + mixed_case_key="$3" + expected_value="$4" + if ! head_object_rest_expect_success_callback "$1" "$2" "" "check_metadata"; then + log 2 "error checking metadata" + return 1 + fi + return 0 +} + +check_metadata() { + log 5 "data: $(cat "$1")" + meta_line=$(grep -i "x-amz-meta" "$1") + log 5 "meta line: $meta_line" + meta_value=$(echo -n "$meta_line" | awk '{print $2}' | sed "s/\r//") + if [ "$meta_value" != "$expected_value" ]; then + log 2 "expected metadata value of '$expected_value', was '$meta_value'" + return 1 + fi + meta_key=$(echo -n "$meta_line" | awk '{print $1}' | sed "s/://") + lowercase_key=$(printf '%s' "$mixed_case_key" | tr '[:upper:]' '[:lower:]') + if [ "$meta_key" != "x-amz-meta-${lowercase_key}" ]; then + log 2 "expected metadata key of '$lowercase_key', was '$meta_key" + return 1 + fi + return 0 +} diff --git a/tests/drivers/head_object/head_object_s3api.sh b/tests/drivers/head_object/head_object_s3api.sh new file mode 100644 index 0000000..e8cdb44 --- /dev/null +++ b/tests/drivers/head_object/head_object_s3api.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +get_and_verify_metadata() { + if [ $# -ne 7 ]; then + log 2 "'get_and_verify_metadata' requires bucket file, expected content type, + expected metadata key, expected metadata val, expected hold status, expected retention mode, expected retention date" + return 1 + fi + if ! head_object "s3api" "$BUCKET_ONE_NAME" "$1"; then + log 2 "error retrieving metadata" + return 1 + fi + # shellcheck disable=SC2154 + raw_metadata=$(echo "$metadata" | grep -v "InsecureRequestWarning") + log 5 "raw metadata: $raw_metadata" + + if ! content_type=$(echo "$raw_metadata" | jq -r ".ContentType" 2>&1); then + log 2 "error retrieving content type: $content_type" + return 1 + fi + if [[ $content_type != "$2" ]]; then + log 2 "content type mismatch ($content_type, $2)" + return 1 + fi + log 5 "metadata key: $3" + if ! meta_val=$(echo "$raw_metadata" | jq -r ".Metadata.$3" 2>&1); then + log 2 "error retrieving metadata val: $meta_val" + return 1 + fi + if [[ $meta_val != "$4" ]]; then + log 2 "metadata val mismatch ($meta_val, $4)" + return 1 + fi + if ! hold_status=$(echo "$raw_metadata" | jq -r ".ObjectLockLegalHoldStatus" 2>&1); then + log 2 "error retrieving hold status: $hold_status" + return 1 + fi + if [[ $hold_status != "$5" ]]; then + log 2 "hold status mismatch ($hold_status, $5)" + return 1 + fi + if ! retention_mode=$(echo "$raw_metadata" | jq -r ".ObjectLockMode" 2>&1); then + log 2 "error retrieving retention mode: $retention_mode" + return 1 + fi + if [[ $retention_mode != "$6" ]]; then + log 2 "retention mode mismatch ($retention_mode, $6)" + return 1 + fi + if ! retain_until_date=$(echo "$raw_metadata" | jq -r ".ObjectLockRetainUntilDate" 2>&1); then + log 2 "error retrieving retain until date: $retain_until_date" + return 1 + fi + if [[ $retain_until_date != "$7"* ]]; then + log 2"retention date mismatch ($retain_until_date, $7)" + return 1 + fi + return 0 +} diff --git a/tests/drivers/not_implemented/not_implemented_rest.sh b/tests/drivers/not_implemented/not_implemented_rest.sh index e901016..72e9391 100644 --- a/tests/drivers/not_implemented/not_implemented_rest.sh +++ b/tests/drivers/not_implemented/not_implemented_rest.sh @@ -33,7 +33,7 @@ send_not_implemented_expect_failure() { fi log 5 "result: $result" echo -n "$result" > "$TEST_FILE_FOLDER/result.txt" - if ! check_rest_expected_header_error "$TEST_FILE_FOLDER/result.txt" "501" "NotImplemented"; then + if ! check_rest_expected_header_error "$TEST_FILE_FOLDER/result.txt" "501" "Not Implemented"; then log 2 "error checking expected header error" return 1 fi diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index e70f902..bc22c53 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -139,34 +139,34 @@ return 0 } send_rest_command_expect_success_callback() { -if ! check_param_count_v2 "env vars, script, response code, callback fn" 4 $#; then - return 1 -fi -output_file="$TEST_FILE_FOLDER/output.txt" -local env_array=("env" "COMMAND_LOG=$COMMAND_LOG" "OUTPUT_FILE=$output_file") -if [ "$1" != "" ]; then - IFS=' ' read -r -a env_vars <<< "$1" - env_array+=("${env_vars[@]}") -fi -# shellcheck disable=SC2068 -if ! result=$(${env_array[@]} "$2" 2>&1); then - log 2 "error sending command: $result" - return 1 -fi -if [ "$result" != "$3" ]; then - log 2 "expected '$3', was '$result' ($(cat "$TEST_FILE_FOLDER/output.txt"))" - return 1 -fi -if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/output.txt"; then - log 2 "callback error" - return 1 -fi -return 0 + if ! check_param_count_v2 "env vars, script, response code, callback fn" 4 $#; then + return 1 + fi + output_file="$TEST_FILE_FOLDER/output.txt" + local env_array=("env" "COMMAND_LOG=$COMMAND_LOG" "OUTPUT_FILE=$output_file") + if [ "$1" != "" ]; then + IFS=' ' read -r -a env_vars <<< "$1" + env_array+=("${env_vars[@]}") + fi + # shellcheck disable=SC2068 + if ! result=$(${env_array[@]} "$2" 2>&1); then + log 2 "error sending command: $result" + return 1 + fi + if [ "$result" != "$3" ]; then + log 2 "expected '$3', was '$result' ($(cat "$TEST_FILE_FOLDER/output.txt"))" + return 1 + fi + if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/output.txt"; then + log 2 "callback error" + return 1 + fi + return 0 } send_rest_go_command_expect_error() { if [ $# -lt 3 ] && [ $(($# % 2)) -ne 1 ]; then - log 2 "'send_rest_go_command_expect_failure' param count must be 3 or greater, odd (key/value pairs)" + log 2 "'send_rest_go_command_expect_error' param count must be 3 or greater, odd (key/value pairs)" return 1 fi if ! curl_command=$(go run ./tests/rest_scripts/generate_command.go -awsAccessKeyId "$AWS_ACCESS_KEY_ID" -awsSecretAccessKey "$AWS_SECRET_ACCESS_KEY" -url "$AWS_ENDPOINT_URL" "${@:4}" 2>&1); then diff --git a/tests/rest_scripts/generate_command.go b/tests/rest_scripts/generate_command.go index ecba383..3417b18 100644 --- a/tests/rest_scripts/generate_command.go +++ b/tests/rest_scripts/generate_command.go @@ -23,6 +23,7 @@ var awsSecretAccessKey *string var serviceName *string var debug *bool var signedParamsMap restParams +var payloadFile *string type S3Command struct { Method string @@ -35,6 +36,7 @@ type S3Command struct { AwsSecretAccessKey string ServiceName string SignedParams map[string]string + PayloadFile string currentDateTime string host string @@ -84,6 +86,7 @@ func main() { AwsSecretAccessKey: *awsSecretAccessKey, ServiceName: *serviceName, SignedParams: signedParamsMap, + PayloadFile: *payloadFile, } curlShellCommand, err := s3Command.CurlShellCommand() if err != nil { @@ -104,6 +107,7 @@ func checkFlags() error { serviceName = flag.String("serviceName", "s3", "Service name") debug = flag.Bool("debug", false, "Print debug statements") flag.Var(&signedParamsMap, "signedParams", "Signed params, separated by comma") + payloadFile = flag.String("payloadFile", "", "Payload file path, if any") // Parse the flags flag.Parse() @@ -168,7 +172,7 @@ func (s *S3Command) generateCanonicalRequestString() { signedParams := []string{"host"} for _, headerValue := range s.headerValues { - key := headerValue[0] + key := strings.ToLower(headerValue[0]) canonicalRequestLines = append(canonicalRequestLines, key+":"+headerValue[1]) signedParams = append(signedParams, key) } @@ -225,5 +229,8 @@ func (s *S3Command) buildCurlShellCommand() string { headerString := fmt.Sprintf("\"%s: %s\"", headerValue[0], headerValue[1]) curlCommand = append(curlCommand, "-H", headerString) } + if s.PayloadFile != "" { + curlCommand = append(curlCommand, "-T", s.PayloadFile) + } return strings.Join(curlCommand, " ") } diff --git a/tests/run.sh b/tests/run.sh index fe4c477..80e5bb5 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -43,6 +43,7 @@ show_help() { echo " rest-list-buckets Run REST list-buckets tests" echo " rest-multipart Run REST multipart tests" echo " rest-not-implemented Run REST multipart tests" + echo " rest-put-object Run REST put-object tests" echo " rest-versioning Run REST versioning tests" echo " rest-bucket Run REST bucket tests" } @@ -57,7 +58,7 @@ handle_param() { s3cmd-file-count|mc|mc-non-file-count|mc-file-count|s3api-user|rest|s3api-policy|\ s3api-bucket|s3api-object|s3api-multipart|rest-base|rest-acl|rest-chunked|rest-checksum|\ rest-create-bucket|rest-head-bucket|rest-list-buckets|rest-not-implemented|\ - rest-versioning|rest-bucket|rest-multipart) + rest-put-object|rest-versioning|rest-bucket|rest-multipart) run_suite "$1" ;; *) # Handle unrecognized options or positional arguments @@ -174,6 +175,8 @@ run_suite() { exit_code=1 elif ! "$HOME"/bin/bats ./tests/test_rest_not_implemented.sh; then exit_code=1 + elif ! "$HOME"/bin/bats ./tests/test_rest_put_object.sh; then + exit_code=1 elif ! "$HOME"/bin/bats ./tests/test_rest_versioning.sh; then exit_code=1 elif ! "$HOME"/bin/bats ./tests/test_rest_bucket.sh; then @@ -220,6 +223,10 @@ run_suite() { echo "Running REST multipart tests ..." "$HOME"/bin/bats ./tests/test_rest_multipart.sh || exit_code=$? ;; + rest-put-object) + echo "Running REST put-object tests ..." + "$HOME"/bin/bats ./tests/test_rest_put_object.sh || exit_code=$? + ;; rest-versioning) echo "Running REST versioning tests ..." "$HOME"/bin/bats ./tests/test_rest_versioning.sh || exit_code=$? diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 1aac8b2..6402046 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -30,13 +30,13 @@ source ./tests/commands/put_object.sh source ./tests/commands/put_object_retention.sh source ./tests/commands/put_object_tagging.sh source ./tests/drivers/copy_object/copy_object_rest.sh +source ./tests/drivers/head_object/head_object_rest.sh source ./tests/drivers/xml.sh source ./tests/logger.sh source ./tests/setup.sh source ./tests/util/util_attributes.sh source ./tests/util/util_chunked_upload.sh source ./tests/util/util_delete_object.sh -source ./tests/util/util_head_object.sh source ./tests/util/util_legal_hold.sh source ./tests/util/util_list_buckets.sh source ./tests/util/util_list_objects.sh @@ -282,43 +282,6 @@ test_file="test_file" assert_success } -@test "REST - put object w/STREAMING-AWS4-HMAC-SHA256-PAYLOAD without content length" { - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" - assert_success - - run put_object_rest_chunked_payload_type_without_content_length "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" - assert_success -} - -@test "REST - invalid 'Expires' parameter" { - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" - assert_success - - run put_object_rest_check_expires_header "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" - assert_success -} - -@test "REST - PutObject with user permission - admin user" { - run setup_bucket_file_and_user "$BUCKET_ONE_NAME" "$test_file" "$USERNAME_ONE" "$PASSWORD_ONE" "admin" - assert_success - username="${lines[${#lines[@]}-2]}" - password="${lines[${#lines[@]}-1]}" - log 5 "username: $username, password: $password" - - run put_object_rest_with_user "$username" "$password" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" - assert_success -} - -@test "REST - PutObject with no permission - 'user' user" { - run setup_bucket_file_and_user "$BUCKET_ONE_NAME" "$test_file" "$USERNAME_ONE" "$PASSWORD_ONE" "user" - assert_success - username="${lines[${#lines[@]}-2]}" - password="${lines[${#lines[@]}-1]}" - - run put_object_rest_with_user_and_code "$username" "$password" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "403" - assert_success -} - @test "REST - PutObject - user permission, bad signature" { run setup_bucket_file_and_user "$BUCKET_ONE_NAME" "$test_file" "$USERNAME_ONE" "$PASSWORD_ONE" "admin" assert_success @@ -375,9 +338,6 @@ test_file="test_file" } @test "REST - PutObjectLegalHold - success" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1193" - fi run setup_bucket_object_lock_enabled "$BUCKET_ONE_NAME" assert_success @@ -427,17 +387,6 @@ test_file="test_file" assert_success } -@test "REST - put object, missing Content-Length" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1321" - fi - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" - assert_success - - run put_object_without_content_length "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file" - assert_success -} - @test "REST - put, get object, encoded name" { file_name=" \"<>\\^\`{}|+&?%" run setup_bucket_and_file "$BUCKET_ONE_NAME" "$file_name" @@ -460,47 +409,9 @@ test_file="test_file" } @test "REST - GetObject w/STREAMING-AWS4-HMAC-SHA256-PAYLOAD type" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1352" - fi run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" assert_success run get_object_rest_with_invalid_streaming_type "$BUCKET_ONE_NAME" "$test_file" assert_success } - -@test "REST - PutObject w/x-amz-checksum-algorithm" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1356" - fi - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" - assert_success - - run put_object_rest_with_unneeded_algorithm_param "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc32c" - assert_success -} - -@test "REST - empty message" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1249" - fi - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" - assert_success - - echo -en "\r\n" > "$TEST_FILE_FOLDER/empty.txt" - run send_via_openssl_with_timeout "$TEST_FILE_FOLDER/empty.txt" - assert_success -} - -@test "REST - deformed message" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1364" - fi - run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" - assert_success - - echo -en "abcdefg\r\n\r\n" > "$TEST_FILE_FOLDER/deformed.txt" - run send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/deformed.txt" 400 "BadRequest" "An error occurred when parsing the HTTP request." - assert_success -} diff --git a/tests/test_rest_acl.sh b/tests/test_rest_acl.sh index 45ed2d3..a5fe2f0 100755 --- a/tests/test_rest_acl.sh +++ b/tests/test_rest_acl.sh @@ -109,9 +109,6 @@ fi } @test "REST - put invalid ACL" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/986" - fi run setup_bucket "$BUCKET_ONE_NAME" assert_success @@ -206,14 +203,3 @@ fi run get_bucket_acl_rest "$BUCKET_ONE_NAME" "" "check_that_acl_xml_does_not_have_owner_permission" assert_success } - -@test "GetBucketAcl - DisplayName" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1414" - fi - run setup_bucket "$BUCKET_ONE_NAME" - assert_success - - run get_bucket_acl_rest "$BUCKET_ONE_NAME" "" "check_for_display_name" - assert_success -} \ No newline at end of file diff --git a/tests/test_rest_bucket.sh b/tests/test_rest_bucket.sh index 668ec1f..8f20f30 100755 --- a/tests/test_rest_bucket.sh +++ b/tests/test_rest_bucket.sh @@ -25,6 +25,7 @@ source ./tests/drivers/list_buckets/list_buckets_rest.sh source ./tests/logger.sh source ./tests/setup.sh source ./tests/util/util_bucket.sh +source ./tests/util/util_delete_object.sh source ./tests/util/util_list_buckets.sh source ./tests/util/util_lock_config.sh source ./tests/util/util_ownership.sh @@ -169,7 +170,7 @@ export RUN_USERS=true @test "REST - POST call on root endpoint" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1036" + skip "https://github.com/versity/versitygw/issues/1487" fi run delete_object_empty_bucket_check_error assert_success diff --git a/tests/test_rest_checksum.sh b/tests/test_rest_checksum.sh index c54e4fc..6691f83 100755 --- a/tests/test_rest_checksum.sh +++ b/tests/test_rest_checksum.sh @@ -18,7 +18,6 @@ load ./bats-support/load load ./bats-assert/load source ./tests/setup.sh -source ./tests/util/util_head_object.sh source ./tests/util/util_setup.sh export RUN_USERS=true diff --git a/tests/test_rest_chunked.sh b/tests/test_rest_chunked.sh index fabdd8d..0c1f6bf 100755 --- a/tests/test_rest_chunked.sh +++ b/tests/test_rest_chunked.sh @@ -22,7 +22,6 @@ source ./tests/setup.sh source ./tests/util/util_bucket.sh source ./tests/util/util_chunked_upload.sh source ./tests/util/util_file.sh -source ./tests/util/util_head_object.sh source ./tests/util/util_setup.sh @test "REST - chunked upload, no content length" { diff --git a/tests/test_rest_create_bucket.sh b/tests/test_rest_create_bucket.sh index a214b91..70bed40 100755 --- a/tests/test_rest_create_bucket.sh +++ b/tests/test_rest_create_bucket.sh @@ -74,9 +74,6 @@ export RUN_USERS=true } @test "REST - CreateBucket - x-amz-grant-full-control - no ownership control change" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1387" - fi if [ "$RECREATE_BUCKETS" == "false" ]; then skip "skip bucket create tests for static buckets" fi diff --git a/tests/test_rest_head_bucket.sh b/tests/test_rest_head_bucket.sh index b2aa678..4dc56ae 100755 --- a/tests/test_rest_head_bucket.sh +++ b/tests/test_rest_head_bucket.sh @@ -41,7 +41,7 @@ source ./tests/setup.sh assert_success } -@test "REST - HeadBucket - success" { +@test "REST - HeadBucket - expected owner success" { run setup_bucket "$BUCKET_ONE_NAME" assert_success diff --git a/tests/test_rest_list_buckets.sh b/tests/test_rest_list_buckets.sh index 176a3e3..3e37646 100755 --- a/tests/test_rest_list_buckets.sh +++ b/tests/test_rest_list_buckets.sh @@ -22,6 +22,32 @@ source ./tests/drivers/list_buckets/list_buckets_rest.sh source ./tests/logger.sh source ./tests/setup.sh +@test "REST - empty message" { + test_file="test_file" + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1249" + fi + run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + echo -en "\r\n" > "$TEST_FILE_FOLDER/empty.txt" + run send_via_openssl_with_timeout "$TEST_FILE_FOLDER/empty.txt" + assert_success +} + +@test "REST - deformed message" { + test_file="test_file" + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1364" + fi + run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + echo -en "abcdefg\r\n\r\n" > "$TEST_FILE_FOLDER/deformed.txt" + run send_via_openssl_check_code_error_contains "$TEST_FILE_FOLDER/deformed.txt" 400 "BadRequest" "An error occurred when parsing the HTTP request." + assert_success +} + @test "test_rest_list_buckets" { run setup_bucket "$BUCKET_ONE_NAME" assert_success diff --git a/tests/test_rest_multipart.sh b/tests/test_rest_multipart.sh index 90c8fa1..d51c483 100755 --- a/tests/test_rest_multipart.sh +++ b/tests/test_rest_multipart.sh @@ -73,27 +73,18 @@ test_file="test_file" } @test "REST - UploadPartCopy w/o upload ID" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1226" - fi run upload_part_copy_without_upload_id_or_part_number "$BUCKET_ONE_NAME" "$test_file" "1" "" \ 400 "InvalidArgument" "This operation does not accept partNumber without uploadId" assert_success } @test "REST - UploadPartCopy w/o part number" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1229" - fi run upload_part_copy_without_upload_id_or_part_number "$BUCKET_ONE_NAME" "$test_file" "" "dummy" \ 405 "MethodNotAllowed" "The specified method is not allowed against this resource" assert_success } @test "REST - UploadPartCopy - ETag is quoted" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1235" - fi run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" assert_success @@ -105,9 +96,6 @@ test_file="test_file" } @test "REST - UploadPart - ETag is quoted" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1233" - fi run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file" assert_success @@ -119,14 +107,11 @@ test_file="test_file" # shellcheck disable=SC2030 upload_id=$output - run upload_part_check_etag_header "$BUCKET_ONE_NAME" "$test_file" "$upload_id" + run upload_part_check_etag_header "$BUCKET_ONE_NAME" "$test_file" "$upload_id" "1" "$TEST_FILE_FOLDER/${test_file}-0" assert_success } @test "REST - UploadPart w/o part number" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1236" - fi run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file" assert_success @@ -206,9 +191,6 @@ test_file="test_file" } @test "REST - multipart - lowercase checksum type and algorithm" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1339" - fi run setup_bucket "$BUCKET_ONE_NAME" assert_success @@ -217,9 +199,6 @@ test_file="test_file" } @test "REST - multipart - full object checksum type doesn't require UploadPart checksums" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1342" - fi run setup_bucket "$BUCKET_ONE_NAME" assert_success @@ -283,9 +262,6 @@ test_file="test_file" } @test "REST - multipart - x-amz-checksum-algorithm is ignored in CompleteMultipartUpload" { - #if [ "$DIRECT" != "true" ]; then - # skip "https://github.com/versity/versitygw/issues/1345" - #fi run test_complete_multipart_upload_unneeded_algorithm_parameter "$BUCKET_ONE_NAME" "$test_file" "FULL_OBJECT" "CRC32C" assert_success } @@ -373,9 +349,6 @@ test_file="test_file" } @test "REST - multipart - x-amz-mp-object-size - invalid string" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/1398" - fi run setup_bucket_and_large_file "$BUCKET_ONE_NAME" "$test_file" assert_success diff --git a/tests/test_rest_not_implemented.sh b/tests/test_rest_not_implemented.sh index 94ae32f..80ce7d8 100755 --- a/tests/test_rest_not_implemented.sh +++ b/tests/test_rest_not_implemented.sh @@ -21,199 +21,166 @@ source ./tests/drivers/not_implemented/not_implemented_rest.sh source ./tests/setup.sh @test "REST - PutBucketAnalyticsConfiguration" { - skip "https://github.com/versity/versitygw/issues/1433" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "analytics=" "PUT" assert_success } @test "REST - Get/ListBucketAnalyticsConfiguration(s)" { - skip "https://github.com/versity/versitygw/issues/1437" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "analytics=" "GET" assert_success } @test "REST - DeleteBucketAnalyticsConfiguration" { - skip "https://github.com/versity/versitygw/issues/1438" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "analytics=" "DELETE" assert_success } @test "REST - GetBucketEncryption" { - skip "https://github.com/versity/versitygw/issues/1439" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "encryption=" "GET" assert_success } @test "REST - PutBucketEncryption" { - skip "https://github.com/versity/versitygw/issues/1439" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "encryption=" "PUT" assert_success } @test "REST - DeleteBucketEncryption" { - skip "https://github.com/versity/versitygw/issues/1439" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "encryption=" "DELETE" assert_success } @test "REST - ListBucketIntelligentTieringConfigurations" { - skip "https://github.com/versity/versitygw/issues/1440" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "intelligent-tiering=" "GET" assert_success } @test "REST - PutBucketIntelligentTieringConfiguration" { - skip "https://github.com/versity/versitygw/issues/1440" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "intelligent-tiering=" "PUT" assert_success } @test "REST - DeleteBucketIntelligentTieringConfiguration" { - skip "https://github.com/versity/versitygw/issues/1440" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "intelligent-tiering=" "DELETE" assert_success } @test "REST - ListBucketInventoryConfigurations" { - skip "https://github.com/versity/versitygw/issues/1441" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "inventory=" "GET" assert_success } @test "REST - PutBucketInventoryConfigurations" { - skip "https://github.com/versity/versitygw/issues/1441" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "inventory=" "PUT" assert_success } @test "REST - DeleteBucketInventoryConfigurations" { - skip "https://github.com/versity/versitygw/issues/1441" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "inventory=" "DELETE" assert_success } @test "REST - GetBucketLifecycleConfiguration" { - skip "https://github.com/versity/versitygw/issues/1443" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "lifecycle=" "GET" assert_success } @test "REST - PutBucketLifecycleConfiguration" { - skip "https://github.com/versity/versitygw/issues/1443" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "lifecycle=" "PUT" assert_success } @test "REST - DeleteBucketLifecycleConfiguration" { - skip "https://github.com/versity/versitygw/issues/1443" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "lifecycle=" "DELETE" assert_success } @test "REST - GetBucketLogging" { - skip "https://github.com/versity/versitygw/issues/1444" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "logging=" "GET" assert_success } @test "REST - PutBucketLogging" { - skip "https://github.com/versity/versitygw/issues/1444" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "logging=" "PUT" assert_success } @test "REST - ListBucketMetricsConfigurations" { - skip "https://github.com/versity/versitygw/issues/1445" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "metrics=" "GET" assert_success } @test "REST - PutBucketMetricsConfigurations" { - skip "https://github.com/versity/versitygw/issues/1445" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "metrics=" "PUT" assert_success } @test "REST - DeleteBucketMetricsConfigurations" { - skip "https://github.com/versity/versitygw/issues/1445" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "metrics=" "DELETE" assert_success } @test "REST - GetBucketReplication" { - skip "https://github.com/versity/versitygw/issues/1449" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "replication=" "GET" assert_success } @test "REST - PutBucketReplication" { - skip "https://github.com/versity/versitygw/issues/1449" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "replication=" "PUT" assert_success } @test "REST - DeleteBucketReplication" { - skip "https://github.com/versity/versitygw/issues/1449" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "replication=" "DELETE" assert_success } @test "REST - GetBucketWebsite" { - skip "https://github.com/versity/versitygw/issues/1450" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "website=" "GET" assert_success } @test "REST - PutBucketWebsite" { - skip "https://github.com/versity/versitygw/issues/1450" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "website=" "PUT" assert_success } @test "REST - DeleteBucketWebsite" { - skip "https://github.com/versity/versitygw/issues/1450" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "website=" "DELETE" assert_success } @test "REST - GetPublicAccessBlock" { - skip "https://github.com/versity/versitygw/issues/1451" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "publicAccessBlock=" "GET" assert_success } @test "REST - PutPublicAccessBlock" { - skip "https://github.com/versity/versitygw/issues/1451" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "publicAccessBlock=" "PUT" assert_success } @test "REST - DeletePublicAccessBlock" { - skip "https://github.com/versity/versitygw/issues/1451" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "publicAccessBlock=" "DELETE" assert_success } @test "REST - GetBucketAccelerateConfiguration" { - skip "https://github.com/versity/versitygw/issues/1452" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "accelerate=" "GET" assert_success } @test "REST - PutBucketAccelerateConfiguration" { - skip "https://github.com/versity/versitygw/issues/1452" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "accelerate=" "PUT" assert_success } @test "REST - GetBucketNotificationConfiguration" { - skip "https://github.com/versity/versitygw/issues/1453" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "notification=" "GET" assert_success } @test "REST - PutBucketNotificationConfiguration" { - skip "https://github.com/versity/versitygw/issues/1453" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "notification=" "PUT" assert_success } @@ -225,13 +192,11 @@ source ./tests/setup.sh } @test "REST - GetBucketRequestPayment" { - skip "https://github.com/versity/versitygw/issues/1455" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "requestPayment=" "GET" assert_success } @test "REST - PutBucketRequestPayment" { - skip "https://github.com/versity/versitygw/issues/1455" run test_not_implemented_expect_failure "$BUCKET_ONE_NAME" "requestPayment=" "PUT" assert_success } diff --git a/tests/test_rest_put_object.sh b/tests/test_rest_put_object.sh new file mode 100755 index 0000000..6ea84ee --- /dev/null +++ b/tests/test_rest_put_object.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +load ./bats-support/load +load ./bats-assert/load + +source ./tests/drivers/head_object/head_object_rest.sh +source ./tests/drivers/put_object/put_object_rest.sh +source ./tests/setup.sh +source ./tests/util/util_setup.sh + +test_file="test_file" +export RUN_USERS=true + +@test "REST - put object w/STREAMING-AWS4-HMAC-SHA256-PAYLOAD without content length" { + run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run put_object_rest_chunked_payload_type_without_content_length "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "REST - PutObject - invalid 'Expires' parameter" { + run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run put_object_rest_check_expires_header "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "REST - PutObject with user permission - admin user" { + run setup_bucket_file_and_user "$BUCKET_ONE_NAME" "$test_file" "$USERNAME_ONE" "$PASSWORD_ONE" "admin" + assert_success + username="${lines[${#lines[@]}-2]}" + password="${lines[${#lines[@]}-1]}" + log 5 "username: $username, password: $password" + + run put_object_rest_with_user "$username" "$password" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success +} + +@test "REST - PutObject with no permission - 'user' user" { + run setup_bucket_file_and_user "$BUCKET_ONE_NAME" "$test_file" "$USERNAME_ONE" "$PASSWORD_ONE" "user" + assert_success + username="${lines[${#lines[@]}-2]}" + password="${lines[${#lines[@]}-1]}" + + run put_object_rest_with_user_and_code "$username" "$password" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "403" + assert_success +} + +@test "REST - put object, missing Content-Length" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1321" + fi + run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run put_object_without_content_length "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file" + assert_success +} + +@test "REST - PutObject w/x-amz-checksum-algorithm" { + run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run put_object_rest_with_unneeded_algorithm_param "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc32c" + assert_success +} + +@test "REST - PutObject - If-None-Match - no asterisk" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/821" + fi + run setup_bucket_and_add_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run send_rest_go_command_expect_error "501" "NotImplemented" "not implemented" "-bucketName" "$BUCKET_ONE_NAME" "-objectKey" "$test_file" "-method" "PUT" "-payloadFile" "$TEST_FILE_FOLDER/$test_file" \ + "-signedParams" "if-none-match:true" + assert_success +} + +@test "REST - PutObject - If-None-Match - block copy" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/821" + fi + test_file="test_file" + run setup_bucket_and_add_file "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run send_rest_go_command_expect_error "412" "PreconditionFailed" "did not hold" "-bucketName" "$BUCKET_ONE_NAME" "-objectKey" "$test_file" "-method" "PUT" "-payloadFile" "$TEST_FILE_FOLDER/$test_file" \ + "-signedParams" "if-none-match:*" + assert_success +} + +@test "REST - PutObject - If-None-Match - success" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/821" + fi + test_file="test_file" + run setup_bucket "$BUCKET_ONE_NAME" + assert_success + + run create_test_files "$test_file" + assert_success + + run send_rest_go_command "200" "-bucketName" "$BUCKET_ONE_NAME" "-objectKey" "$test_file" "-method" "PUT" "-payloadFile" "$TEST_FILE_FOLDER/$test_file" \ + "-signedParams" "if-none-match:*" + assert_success +} + +@test "REST - PutObject - If-Match - file doesn't exist on server" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/821" + fi + test_file="test_file" + run setup_bucket "$BUCKET_ONE_NAME" + assert_success + + run send_rest_go_command_expect_error "404" "NoSuchKey" "key does not exist" "-bucketName" "$BUCKET_ONE_NAME" "-objectKey" "$test_file" "-method" "PUT" "-payloadFile" "$TEST_FILE_FOLDER/$test_file" \ + "-signedParams" "if-match:abc" + assert_success +} + +@test "REST - PutObject - If-Match - incorrect etag" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/821" + fi + test_file="test_file" + run setup_bucket "$BUCKET_ONE_NAME" + assert_success + + run create_test_files "$test_file" + assert_success + + run put_object_rest "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run send_rest_go_command_expect_error "412" "PreconditionFailed" "did not hold" "-bucketName" "$BUCKET_ONE_NAME" "-objectKey" "$test_file" "-method" "PUT" "-payloadFile" "$TEST_FILE_FOLDER/$test_file" \ + "-signedParams" "if-match:abc" + assert_success +} + +@test "REST - PutObject - If-Match - correct etag" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/821" + fi + test_file="test_file" + run setup_bucket "$BUCKET_ONE_NAME" + assert_success + + run create_test_files "$test_file" + assert_success + + run put_object_rest "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run get_etag_rest "$BUCKET_ONE_NAME" "$test_file" + assert_success + etag=${lines[${#lines[@]}-1]} + log 5 "etag: $etag" + + run send_rest_go_command "200" "-bucketName" "$BUCKET_ONE_NAME" "-objectKey" "$test_file" "-method" "PUT" "-payloadFile" "$TEST_FILE_FOLDER/$test_file" \ + "-signedParams" "if-match:$etag" + assert_success +} + +@test "PutObject - metadata keys are made lowercase" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1482" + fi + uppercase_key="CAPITAL" + uppercase_value="DUMMY" + + test_file="test_file" + run setup_bucket "$BUCKET_ONE_NAME" + assert_success + + run create_test_files "$test_file" + assert_success + + run send_rest_go_command "200" "-bucketName" "$BUCKET_ONE_NAME" "-objectKey" "$test_file" "-method" "PUT" "-payloadFile" "$TEST_FILE_FOLDER/$test_file" \ + "-signedParams" "x-amz-meta-$uppercase_key:$uppercase_value" + assert_success + + run check_metadata_key_case "$BUCKET_ONE_NAME" "$test_file" "$uppercase_key" "$uppercase_value" + assert_success +} diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index 4fa2809..9d8e75e 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -28,6 +28,7 @@ source ./tests/util/util_tags.sh source ./tests/commands/get_object.sh source ./tests/commands/put_object.sh source ./tests/commands/list_multipart_uploads.sh +source ./tests/drivers/head_object/head_object_s3api.sh export RUN_USERS=true diff --git a/tests/test_s3api_root_inner.sh b/tests/test_s3api_root_inner.sh index 7898724..2084374 100755 --- a/tests/test_s3api_root_inner.sh +++ b/tests/test_s3api_root_inner.sh @@ -20,7 +20,6 @@ source ./tests/commands/list_parts.sh source ./tests/util/util_get_bucket_acl.sh source ./tests/util/util_get_object_attributes.sh source ./tests/util/util_get_object_retention.sh -source ./tests/util/util_head_object.sh source ./tests/util/util_legal_hold.sh source ./tests/util/util_list_objects.sh source ./tests/util/util_setup.sh diff --git a/tests/util/util_head_object.sh b/tests/util/util_head_object.sh deleted file mode 100644 index d4cbae2..0000000 --- a/tests/util/util_head_object.sh +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2024 Versity Software -# This file is licensed under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -get_and_verify_metadata() { - if [ $# -ne 7 ]; then - log 2 "'get_and_verify_metadata' requires bucket file, expected content type, - expected metadata key, expected metadata val, expected hold status, expected retention mode, expected retention date" - return 1 - fi - if ! head_object "s3api" "$BUCKET_ONE_NAME" "$1"; then - log 2 "error retrieving metadata" - return 1 - fi - # shellcheck disable=SC2154 - raw_metadata=$(echo "$metadata" | grep -v "InsecureRequestWarning") - log 5 "raw metadata: $raw_metadata" - - if ! content_type=$(echo "$raw_metadata" | jq -r ".ContentType" 2>&1); then - log 2 "error retrieving content type: $content_type" - return 1 - fi - if [[ $content_type != "$2" ]]; then - log 2 "content type mismatch ($content_type, $2)" - return 1 - fi - if ! meta_val=$(echo "$raw_metadata" | jq -r ".Metadata.$3" 2>&1); then - log 2 "error retrieving metadata val: $meta_val" - return 1 - fi - if [[ $meta_val != "$4" ]]; then - log 2 "metadata val mismatch ($meta_val, $4)" - return 1 - fi - if ! hold_status=$(echo "$raw_metadata" | jq -r ".ObjectLockLegalHoldStatus" 2>&1); then - log 2 "error retrieving hold status: $hold_status" - return 1 - fi - if [[ $hold_status != "$5" ]]; then - log 2 "hold status mismatch ($hold_status, $5)" - return 1 - fi - if ! retention_mode=$(echo "$raw_metadata" | jq -r ".ObjectLockMode" 2>&1); then - log 2 "error retrieving retention mode: $retention_mode" - return 1 - fi - if [[ $retention_mode != "$6" ]]; then - log 2 "retention mode mismatch ($retention_mode, $6)" - return 1 - fi - if ! retain_until_date=$(echo "$raw_metadata" | jq -r ".ObjectLockRetainUntilDate" 2>&1); then - log 2 "error retrieving retain until date: $retain_until_date" - return 1 - fi - if [[ $retain_until_date != "$7"* ]]; then - log 2"retention date mismatch ($retain_until_date, $7)" - return 1 - fi - return 0 -} - -get_etag_rest() { - if [ $# -ne 2 ]; then - log 2 "'get_etag_rest' requires bucket name, object key" - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/head_object.txt" ./tests/rest_scripts/head_object.sh); then - log 2 "error attempting to get object info: $result" - return 1 - fi - if [ "$result" != "200" ]; then - log 2 "response code '$result', data: $(cat "$TEST_FILE_FOLDER/head_object.txt")" - return 1 - fi - log 5 "head object data: $(cat "$TEST_FILE_FOLDER/head_object.txt")" - etag_value=$(grep "E[Tt]ag:" "$TEST_FILE_FOLDER/head_object.txt" | sed -n 's/E[Tt]ag: "\([^"]*\)"/\1/p' | tr -d '\r') - echo "$etag_value" -} - -verify_object_not_found() { - if [ $# -ne 2 ]; then - log 2 "'verify_object_not_found' requires bucket name, object key" - return 1 - fi - if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" ./tests/rest_scripts/head_object.sh); then - log 2 "error getting result: $result" - return 1 - fi - if [ "$result" != "404" ]; then - log 2 "expected '404', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" - return 1 - fi - return 0 -} - -verify_object_exists() { - if [ $# -ne 2 ]; then - log 2 "'verify_object_not_found' requires bucket name, object key" - return 1 - fi - if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" ./tests/rest_scripts/head_object.sh); then - log 2 "error getting result: $result" - return 1 - fi - if [ "$result" != "200" ]; then - log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/result.txt"))" - return 1 - fi - return 0 -} - -check_checksum_rest() { - if [ $# -ne 4 ]; then - log 2 "'check_checksum_rest' requires bucket, file, expected checksum, header key" - return 1 - fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" CHECKSUM="true" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/head_object.sh 2>&1); then - log 2 "error: $result" - return 1 - fi - if [ "$result" != "200" ]; then - log 2 "expected response code '200', was '$result'" - return 1 - fi - head_checksum=$(grep -i "$4" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//') - if [ "$3" != "$head_checksum" ]; then - log 2 "'checksum mismatch (head '$head_checksum', local '$file_checksum')" - return 1 - fi -} - -check_checksum_rest_sha256() { - if [ $# -ne 3 ]; then - log 2 "'check_checksum_rest_sha256' requires bucket, file, local file" - return 1 - fi - file_checksum="$(sha256sum "$3" | awk '{print $1}' | xxd -r -p | base64)" - if ! check_checksum_rest "$1" "$2" "$file_checksum" "x-amz-checksum-sha256"; then - log 2 "error checking checksum" - return 1 - fi - return 0 -} - -check_checksum_rest_crc32() { - if [ $# -ne 3 ]; then - log 2 "'check_checksum_rest_crc32' requires bucket, file, local file" - return 1 - fi - file_checksum="$(gzip -c -1 "$3" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)" - if ! check_checksum_rest "$1" "$2" "$file_checksum" "x-amz-checksum-crc32"; then - log 2 "error checking checksum" - return 1 - fi - return 0 -} - -head_object_without_and_with_checksum() { - if [ $# -ne 2 ]; then - log 2 "'head_object_without_checksum' requires bucket, file" - return 1 - fi - if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" ./tests/rest_scripts/head_object.sh); then - log 2 "error getting result: $result" - return 1 - fi - head_checksum=$(grep -i "x-amz-checksum-sha256" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//') - if [ "$head_checksum" != "" ]; then - log 2 "head checksum shouldn't be returned, is $head_checksum" - return 1 - fi - if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" CHECKSUM="true" ./tests/rest_scripts/head_object.sh); then - log 2 "error getting result: $result" - return 1 - fi - head_checksum=$(grep -i "x-amz-checksum-sha256" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//') - if [ "$head_checksum" == "" ]; then - log 2 "head checksum should be returned" - return 1 - fi - return 0 -} - -check_default_checksum() { - if [ $# -ne 3 ]; then - log 2 "'head_object_without_checksum' requires bucket, file, local file" - return 1 - fi - if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" ./tests/rest_scripts/head_object.sh); then - log 2 "error getting result: $result" - return 1 - fi - if ! result=$(OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OBJECT_KEY="$2" CHECKSUM="true" ./tests/rest_scripts/head_object.sh); then - log 2 "error getting result: $result" - return 1 - fi - log 5 "result: $(cat "$TEST_FILE_FOLDER/result.txt")" - head_checksum=$(grep -i "x-amz-checksum-crc64nvme" "$TEST_FILE_FOLDER/result.txt" | awk '{print $2}' | sed 's/\r$//') - log 5 "checksum: $head_checksum" - if ! checksum=$(CHECKSUM_TYPE="crc64nvme" DATA_FILE="$3" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_checksum.sh); then - log 2 "error calculating local checksum: $checksum" - return 1 - fi - if [ "$head_checksum" != "$checksum" ]; then - log 2 "checksum mismatch (returned: '$head_checksum', local: '$checksum')" - return 1 - fi - return 0 -} - -get_object_size_with_user() { - if ! check_param_count "get_object_size_with_user" "username, password, bucket, key" 4 $#; then - return 1 - fi - if ! result=$(AWS_ACCESS_KEY_ID="$1" AWS_SECRET_ACCESS_KEY="$2" COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$3" OBJECT_KEY="$4" OUTPUT_FILE="$TEST_FILE_FOLDER/head_object.txt" ./tests/rest_scripts/head_object.sh 2>&1); then - log 2 "error attempting to get object info: $result" - return 1 - fi - if [ "$result" != "200" ]; then - log 2 "response code '$result', data: $(cat "$TEST_FILE_FOLDER/head_object.txt")" - return 1 - fi - log 5 "head object data: $(cat "$TEST_FILE_FOLDER/head_object.txt")" - content_length=$(grep "Content-Length:" "$TEST_FILE_FOLDER/head_object.txt" | awk '{print $2}' | tr -d '\r') - log 5 "file size: $content_length" - echo "$content_length" -} diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh index 6412047..dc0bb32 100644 --- a/tests/util/util_multipart.sh +++ b/tests/util/util_multipart.sh @@ -301,13 +301,19 @@ setup_multipart_upload_with_params() { return 1 fi os_name="$(uname)" + if [ "$DIRECT" == "true" ]; then + later_seconds="40" + else + later_seconds="20" + fi if [[ "$os_name" == "Darwin" ]]; then now=$(date -u +"%Y-%m-%dT%H:%M:%S") - later=$(date -j -v +20S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S") + later=$(date -j -v "+${later_seconds}S" -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S") else now=$(date +"%Y-%m-%dT%H:%M:%S") - later=$(date -d "$now 20 seconds" +"%Y-%m-%dT%H:%M:%S") + later=$(date -d "$now $later_seconds seconds" +"%Y-%m-%dT%H:%M:%S") fi + log 5 "later in function: $later" if ! create_test_files "$2"; then log 2 "error creating test file" diff --git a/tests/util/util_multipart_before_completion.sh b/tests/util/util_multipart_before_completion.sh index ca37ffc..b1fd421 100644 --- a/tests/util/util_multipart_before_completion.sh +++ b/tests/util/util_multipart_before_completion.sh @@ -435,11 +435,10 @@ upload_part_copy_without_upload_id_or_part_number() { } upload_part_check_etag_header() { - if [ $# -ne 3 ]; then - log 2 "'upload_part_check_etag_header' requires bucket name, key, upload ID" + if ! check_param_count_v2 "bucket, key, upload ID, part number, part" 5 $#; then return 1 fi - if ! etag=$(upload_part_rest "$1" "$2" "$3" 1 2>&1); then + if ! etag=$(upload_part_rest "$1" "$2" "$3" "$4" "$5" 2>&1); then log 2 "error getting etag: $etag" return 1 fi diff --git a/tests/util/util_object.sh b/tests/util/util_object.sh index 597a637..9669a2a 100644 --- a/tests/util/util_object.sh +++ b/tests/util/util_object.sh @@ -16,7 +16,6 @@ source ./tests/util/util_bucket.sh source ./tests/util/util_create_bucket.sh -source ./tests/util/util_head_object.sh source ./tests/util/util_mc.sh source ./tests/util/util_multipart.sh source ./tests/util/util_versioning.sh @@ -46,6 +45,7 @@ source ./tests/commands/put_object_legal_hold.sh source ./tests/commands/put_object_lock_configuration.sh source ./tests/commands/upload_part_copy.sh source ./tests/commands/upload_part.sh +source ./tests/drivers/head_object/head_object_rest.sh source ./tests/util/util_users.sh # params: bucket, object name @@ -243,7 +243,7 @@ check_checksum_invalid_or_incorrect() { put_object_rest_checksum() { log 6 "put_object_rest_checksum" - if ! check_param_count "put_object_rest_checksum" "data file, bucket name, key, checksum type" 4 $#; then + if ! check_param_count_v2 "data file, bucket name, key, checksum type" 4 $#; then return 1 fi # shellcheck disable=SC2097,SC2098