diff --git a/tests/drivers/list_buckets/list_buckets_rest.sh b/tests/drivers/list_buckets/list_buckets_rest.sh index c25d0e4e..aa46e844 100644 --- a/tests/drivers/list_buckets/list_buckets_rest.sh +++ b/tests/drivers/list_buckets/list_buckets_rest.sh @@ -265,3 +265,51 @@ list_check_buckets_user() { fi return 0 } + +verify_bucket_not_in_list() { + if ! check_param_count_v2 "data file, bucket name" 2 $#; then + return 1 + fi + if check_xml_element "$1" "$2" "ListAllMyBucketsResult" "Buckets" "Bucket" "Name"; then + log 2 "bucket shouldn't be returned in list" + return 1 + fi + return 0 +} + +list_buckets_bucket_not_in_list() { + if ! check_param_count_gt "bucket name, region, other params" 2 $#; then + return 1 + fi + if ! send_rest_go_command_callback "200" "verify_bucket_not_in_list" "-query" "bucket-region=$2" "${@:3}" "--" "$1"; then + log 2 "error verifying that bucket '$1' is not returned" + return 1 + fi + return 0 +} + +check_bucket_and_region() { + if ! check_param_count_v2 "data file, bucket, region" 3 $#; then + return 1 + fi + if ! bucket_info=$(get_element_with_matching_inner_value "$1" "$2" "ListAllMyBucketsResult" "Buckets" "Bucket" "--" "Name" 2>&1); then + log 2 "error getting bucket with name '$1': $bucket_info" + return 1 + fi + if ! check_xml_element_inside_string "$bucket_info" "$3" "Bucket" "BucketRegion"; then + log 2 "error checking for bucket region '$3' inside bucket info" + return 1 + fi + return 0 +} + +list_check_bucket_and_region() { + if ! check_param_count_v2 "bucket, region" 2 $#; then + return 1 + fi + if ! send_rest_go_command_callback "200" "check_bucket_and_region" "-query" "bucket-region=$2" \ + "--" "$1" "$2"; then + return 1 + fi + return 0 +} diff --git a/tests/drivers/rest.sh b/tests/drivers/rest.sh index 95affcfc..3c0f4889 100644 --- a/tests/drivers/rest.sh +++ b/tests/drivers/rest.sh @@ -147,7 +147,7 @@ send_rest_command_expect_success_callback() { if ! check_param_count_v2 "env vars, script, response code, callback fn" 4 $#; then return 1 fi - if ! output_file_name=$(get_file_name); then + if ! output_file_name=$(get_file_name 2>&1); then log 2 "error generating output file name: $output_file_name" return 1 fi @@ -186,6 +186,7 @@ rest_go_command_perform_send() { return 1 fi log 5 "result: $result" + echo "$result" } send_rest_go_command_expect_error() { @@ -205,16 +206,34 @@ send_rest_go_command_expect_error_callback() { log 2 "'send_rest_go_command_expect_error' param count must be 4 or greater, even (expected HTTP code, expected error code, expected message, callback, go params)" return 1 fi - if ! rest_go_command_perform_send "${@:5}"; then - log 2 "error sending rest go command" + + local all_params=("${@:5}") no_callback_params=0 go_param_array=() callback_params=() + + if ! params_file=$(get_file_name 2>&1); then + log 2 "error getting params file name: $params_file" return 1 fi - echo -n "$result" > "$TEST_FILE_FOLDER/result.txt" - if ! check_rest_go_expected_error "$TEST_FILE_FOLDER/result.txt" "$1" "$2" "$3"; then + get_go_params "${all_params[@]}" > "$TEST_FILE_FOLDER/$params_file" || no_callback_params=$? + mapfile -t go_param_array < "$TEST_FILE_FOLDER/$params_file" + + if [ "$no_callback_params" -eq 1 ]; then + mapfile -t callback_params < <(get_callback_params "${all_params[@]}") + fi + + if ! result=$(rest_go_command_perform_send "${go_param_array[@]}" 2>&1); then + log 2 "error sending rest go command: $result" + return 1 + fi + if ! file_name=$(get_file_name 2>&1); then + log 2 "error getting file name: $file_name" + return 1 + fi + echo -n "$result" > "$TEST_FILE_FOLDER/$file_name" + if ! check_rest_go_expected_error "$TEST_FILE_FOLDER/$file_name" "$1" "$2" "$3"; then log 2 "error checking expected header error" return 1 fi - if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/result.txt"; then + if [ "$4" != "" ] && ! "$4" "$TEST_FILE_FOLDER/$file_name" "${callback_params[@]}"; then log 2 "callback error" return 1 fi @@ -283,15 +302,17 @@ send_rest_go_command_callback() { return 1 fi - local all_params=("${@:3}") no_callback_params=0 go_params go_param_array=() callback_params=() - go_params=$(get_go_params "${all_params[@]}") || no_callback_params=$? - while IFS= read -r line; do - go_param_array+=("$line") - done <<< "$go_params" + local all_params=("${@:3}") no_callback_params=0 go_param_array=() callback_params=() + + if ! params_file=$(get_file_name 2>&1); then + log 2 "error getting params file name: $params_file" + return 1 + fi + get_go_params "${all_params[@]}" > "$TEST_FILE_FOLDER/$params_file" || no_callback_params=$? + mapfile -t go_param_array < "$TEST_FILE_FOLDER/$params_file" + if [ "$no_callback_params" -eq 1 ]; then - while IFS= read -r line; do - callback_params+=("$line") - done <<< "$(get_callback_params "${all_params[@]}")" + mapfile -t callback_params < <(get_callback_params "${all_params[@]}") fi if ! rest_go_command_perform_send "${go_param_array[@]}"; then @@ -379,11 +400,11 @@ send_rest_go_command_expect_error_with_arg_name_value() { } check_specific_argument_name_and_value() { - if ! check_param_count_v2 "data file" 1 $#; then + if ! check_param_count_v2 "data file, argument name, value" 3 $#; then return 1 fi - if ! check_error_parameter "$1" "$argument_name" "$argument_value"; then - log 2 "error checking '$argument_name' parameter" + if ! check_error_parameter "$1" "$2" "$3"; then + log 2 "error checking '$2' parameter" return 1 fi } @@ -392,9 +413,7 @@ send_rest_go_command_expect_error_with_specific_arg_name_value() { if ! check_param_count_gt "response code, error code, message, arg name, arg value, params" 5 $#; then return 1 fi - argument_name=$4 - argument_value=$5 - if ! send_rest_go_command_expect_error_callback "$1" "$2" "$3" "check_specific_argument_name_and_value" "${@:6}"; then + if ! send_rest_go_command_expect_error_callback "$1" "$2" "$3" "check_specific_argument_name_and_value" "${@:6}" "--" "$4" "$5"; then log 2 "error checking error response values" return 1 fi diff --git a/tests/drivers/xml.sh b/tests/drivers/xml.sh index a24cf8a8..d8eee1f2 100644 --- a/tests/drivers/xml.sh +++ b/tests/drivers/xml.sh @@ -24,25 +24,57 @@ build_xpath_string() { xpath+='/text()' } +get_xpath_segment() { + if ! check_param_count_gt "XML element name" 1 $#; then + return 1 + fi + if [ "$1" == "" ]; then + log 2 "element has no name" + return 1 + fi + if [[ "$1" =~ [[:space:]] ]]; then + log 2 "element '$1' contains a space" + return 1 + fi + echo '*[local-name()="'"$1"'"]' +} + + build_xpath_string_for_element() { if ! check_param_count_gt "XML tree" 1 $#; then return 1 fi local xpath='//' for ((idx=1;idx<=$#;idx++)); do - if [ "${!idx}" == "" ]; then - log 2 "param number $idx is empty" + if ! segment=$(get_xpath_segment "${!idx}" 2>&1); then + log 2 "error getting xpath segment: $segment" return 1 fi - if [[ "${!idx}" =~ [[:space:]] ]]; then - log 2 "param '${!idx}' contains a space" - return 1 - fi - xpath+='*[local-name()="'${!idx}'"]' + xpath+="$segment" if [ "$idx" != $# ]; then xpath+='/' fi done + log 5 "xpath: $xpath" + echo "$xpath" + return 0 +} + +get_inner_xpath_string_for_element() { + if ! check_param_count_gt "compare element, XML tree" 2 $#; then + return 1 + fi + local xpath='[' + for ((idx=2;idx<=$#;idx++)); do + if ! xpath+=$(get_xpath_segment "${!idx}" 2>&1); then + log 2 "error getting xpath segment: $xpath" + return 1 + fi + if [ "$idx" != $# ]; then + xpath+='/' + fi + done + xpath+="='$1']" echo "$xpath" return 0 } @@ -130,6 +162,22 @@ check_xml_element() { return 0 } +check_xml_element_inside_string() { + if ! check_param_count_gt "string, expected value, XML tree" 3 $#; then + return 1 + fi + if ! data_file=$(get_file_name 2>&1); then + log 2 "error getting data file: $data_file" + return 1 + fi + echo -n "$1" > "$TEST_FILE_FOLDER/$data_file" + if ! check_xml_element "$TEST_FILE_FOLDER/$data_file" "$2" "${@:3}"; then + log 2 "error checking XML element" + return 1 + fi + return 0 +} + check_xml_element_contains() { if [ $# -lt 3 ]; then log 2 "'check_xml_element_contains' requires data source, expected value, XML tree" @@ -210,6 +258,22 @@ check_if_element_exists() { return 1 } +print_xml_data_to_file() { + if ! check_param_count_v2 "data file" 1 $#; then + return 1 + fi + if ! file_name=$(get_file_name 2>&1); then + log 2 "error getting file name: $file_name" + return 1 + fi + if ! get_xml_data "$1" "$TEST_FILE_FOLDER/$file_name"; then + log 2 "error getting xml data" + return 1 + fi + echo "$TEST_FILE_FOLDER/$file_name" + return 0 +} + get_xml_data() { if ! check_param_count_v2 "data file, output file" 2 $#; then return 1 @@ -287,3 +351,45 @@ compare_data_with_xml_file() { fi return 0 } + +get_element_with_matching_inner_value() { + if ! check_param_count_gt "data file, matching value, outer value, with inner value separated by '--'" 4 $#; then + return 1 + fi + local outer_params=() inner_params=() separator_found=false + for param in "${@:3}"; do + if [ "$param" == "--" ]; then + separator_found=true + continue + fi + if [ "$separator_found" == "false" ]; then + outer_params+=("$param") + continue + fi + inner_params+=("$param") + done + if [ "${#outer_params}" -eq 0 ] || [ "${#inner_params}" -eq 0 ]; then + log 2 "command requires params separated by '--'" + return 1 + fi + if ! xml_data_file=$(print_xml_data_to_file "$1" 2>&1); then + log 2 "error writing XML to data file: $xml_data_file" + return 1 + fi + if ! xpath=$(build_xpath_string_for_element "${outer_params[@]}" 2>&1); then + log 2 "error getting outer segment: $xpath" + return 1 + fi + if ! inner_xpath=$(get_inner_xpath_string_for_element "$2" "${inner_params[@]}" 2>&1); then + log 2 "error getting inner segment: $inner_xpath" + return 1 + fi + xpath+="$inner_xpath" + log 5 "full xpath: $xpath" + if ! result=$(xmllint --xpath "${xpath}" "$xml_data_file" 2>&1); then + log 2 "error getting result: $result" + return 1 + fi + echo "$result" + return 0 +} diff --git a/tests/rest_scripts/command/s3Command.go b/tests/rest_scripts/command/s3Command.go index 79a2d9df..9f326c2d 100644 --- a/tests/rest_scripts/command/s3Command.go +++ b/tests/rest_scripts/command/s3Command.go @@ -81,6 +81,10 @@ type S3Command struct { OmitPayloadTrailer bool OmitPayloadTrailerKey bool OmitContentLength bool + OmitSHA256Hash bool + CustomSHA256Hash string + OmitDate bool + CustomDate string dataSource DataSource currentDateTime string @@ -116,7 +120,9 @@ func (s *S3Command) CurlShellCommand() (string, error) { func (s *S3Command) prepareForBuild() error { now := time.Now().UTC() - if s.IncorrectYearMonthDay { + if s.CustomDate != "" { + s.currentDateTime = s.CustomDate + } else if s.IncorrectYearMonthDay { s.currentDateTime = now.Add(-48 * time.Hour).Format("20060102T150405Z") } else { s.currentDateTime = now.Format("20060102T150405Z") @@ -151,7 +157,9 @@ func (s *S3Command) preparePayload() error { } else if s.Payload != "" { s.dataSource = NewStringDataSource(s.Payload) } - if s.PayloadType != "" { + if s.CustomSHA256Hash != "" { + s.payloadHash = s.CustomSHA256Hash + } else if s.PayloadType != "" { s.payloadHash = s.PayloadType } else if s.dataSource != nil { var err error @@ -193,8 +201,7 @@ func (s *S3Command) initializeOpenSSLPayloadAndGetContentLength() error { return nil } -func (s *S3Command) addHeaderValues() error { - s.headerValues = []*HeaderValue{} +func (s *S3Command) addBaseHeaderValues() { if s.MissingHostParam { s.headerValues = append(s.headerValues, &HeaderValue{"host", "", true}) } else if s.CustomHostParamSet { @@ -202,17 +209,26 @@ func (s *S3Command) addHeaderValues() error { } else { s.headerValues = append(s.headerValues, &HeaderValue{"host", s.host, true}) } - if s.PayloadType == StreamingAWS4HMACSHA256PayloadTrailer && s.ChecksumType != "" { - s.headerValues = append(s.headerValues, &HeaderValue{"x-amz-trailer", fmt.Sprintf("x-amz-checksum-%s", s.ChecksumType), true}) + if !s.OmitSHA256Hash { + s.headerValues = append(s.headerValues, &HeaderValue{"x-amz-content-sha256", s.payloadHash, true}) + } + if !s.OmitDate { + s.headerValues = append(s.headerValues, &HeaderValue{"x-amz-date", s.currentDateTime, true}) } - s.headerValues = append(s.headerValues, - &HeaderValue{"x-amz-content-sha256", s.payloadHash, true}, - &HeaderValue{"x-amz-date", s.currentDateTime, true}, - ) if s.Client == OPENSSL && !s.OmitContentLength { s.headerValues = append(s.headerValues, &HeaderValue{"Content-Length", fmt.Sprintf("%d", s.contentLength), true}) } +} + +func (s *S3Command) addHeaderValues() error { + s.headerValues = []*HeaderValue{} + + s.addBaseHeaderValues() + + if s.PayloadType == StreamingAWS4HMACSHA256PayloadTrailer && s.ChecksumType != "" { + s.headerValues = append(s.headerValues, &HeaderValue{"x-amz-trailer", fmt.Sprintf("x-amz-checksum-%s", s.ChecksumType), true}) + } if s.dataSource != nil && s.PayloadType != UnsignedPayload { payloadSize, err := s.dataSource.SourceDataByteSize() if err != nil { diff --git a/tests/rest_scripts/generateCommand.go b/tests/rest_scripts/generateCommand.go index 0b8560b5..44de1209 100644 --- a/tests/rest_scripts/generateCommand.go +++ b/tests/rest_scripts/generateCommand.go @@ -48,6 +48,8 @@ var customHostParam *string var customHostParamSet bool = false var commandType *string var checksumType *string +var customSHA256Hash *string +var customDate *string type arrayFlags []string @@ -61,6 +63,8 @@ var chunkSize *int var omitPayloadTrailer *bool var omitPayloadTrailerKey *bool var omitContentLength *bool +var omitSHA256Hash *bool +var omitDate *bool var locationConstraint *string var locationConstraintSet bool = false @@ -138,7 +142,11 @@ func main() { OmitPayloadTrailer: *omitPayloadTrailer, OmitPayloadTrailerKey: *omitPayloadTrailerKey, OmitContentLength: *omitContentLength, + OmitSHA256Hash: *omitSHA256Hash, + CustomSHA256Hash: *customSHA256Hash, Client: *client, + OmitDate: *omitDate, + CustomDate: *customDate, } s3Command, err := getS3CommandType(baseCommand) @@ -245,6 +253,10 @@ func checkFlags() error { omitPayloadTrailer = flag.Bool("omitPayloadTrailer", false, "Omit final trailer for chunked uploads w/trailers") omitPayloadTrailerKey = flag.Bool("omitPayloadTrailerKey", false, "Omit final trailer key for chunked uploads w/trailer") omitContentLength = flag.Bool("omitContentLength", false, "Omit content length parameter") + omitSHA256Hash = flag.Bool("omitSHA256Hash", false, "Omit sha256 hash parameter") + customSHA256Hash = flag.String("customSHA256Hash", "", "Add a custom sha256 hash value") + omitDate = flag.Bool("omitDate", false, "Omit x-amz-date parameter") + customDate = flag.String("customDate", "", "Use a custom x-amz-date parameter") flag.Var(&tagKeys, "tagKey", "Tag key (can add multiple)") flag.Var(&tagValues, "tagValue", "Tag value (can add multiple)") locationConstraint = flag.String("locationConstraint", "", "Location constraint for bucket creation") diff --git a/tests/test_rest_create_bucket.sh b/tests/test_rest_create_bucket.sh index ae28027e..72d875be 100755 --- a/tests/test_rest_create_bucket.sh +++ b/tests/test_rest_create_bucket.sh @@ -128,7 +128,7 @@ export RUN_USERS=true } @test "REST - CreateBucket - location constraint mismatch" { - if [ "$DIRECT" != "true" ]; then + if [ "$DIRECT" == "true" ]; then skip "not valid for direct mode" fi local region="us-east-1" diff --git a/tests/test_rest_list_buckets.sh b/tests/test_rest_list_buckets.sh index 6efe90c2..5e9349b5 100755 --- a/tests/test_rest_list_buckets.sh +++ b/tests/test_rest_list_buckets.sh @@ -237,4 +237,124 @@ export RUN_USERS=true fi run send_rest_go_command_check_header_key_and_value "400" "Content-Type" "application/xml" "-method" "GETS" assert_success -} \ No newline at end of file +} + +@test "REST - ListBuckets - invalid bucket-region query" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1923" + fi + local invalid_region="abc" + + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + local bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error_with_specific_arg_name_value "400" "InvalidArgument" "Argument value $invalid_region is not a valid AWS Region" \ + "ArgumentName" "bucket-region" "-query" "bucket-region=$invalid_region" + assert_success +} + +@test "REST - ListBuckets - incorrect bucket region" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1930" + fi + local test_region="us-east-1" + if [ "$AWS_REGION" == "us-east-1" ]; then + test_region="us-west-1" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + local params=() + if [ "$DIRECT" == "true" ]; then + params+=("-url" "https://s3.$test_region.amazonaws.com" "-awsRegion" "$test_region") + fi + + run list_buckets_bucket_not_in_list "$bucket_name" "$test_region" "${params[@]}" + assert_success +} + +@test "REST - ListBuckets - correct bucket region" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_callback "200" "check_bucket_and_region" "-query" "bucket-region=$AWS_REGION" "--" "$bucket_name" "$AWS_REGION" + assert_success +} + +@test "REST - ListBuckets - missing sha256 hash" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "400" "InvalidRequest" "Missing required header for this request: x-amz-content-sha256" "-omitSHA256Hash" + assert_success +} + +@test "REST - ListBuckets - invalid hash type" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "400" "InvalidArgument" "x-amz-content-sha256 must be" "-customSHA256Hash" "ABCDEFG" + assert_success +} + +@test "REST - ListBuckets - non-matching hash type" { + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "400" "InvalidRequest" "The value of x-amz-content-sha256 header is invalid" "-customSHA256Hash" "STREAMING-UNSIGNED-PAYLOAD-TRAILER" + assert_success +} + +@test "REST - ListBuckets - omit date" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1934" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "403" "AccessDenied" "AWS authentication requires a valid Date or x-amz-date header" "-omitDate" + assert_success +} + +@test "REST - ListBuckets - invalid date" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/1934" + fi + run get_bucket_name "$BUCKET_ONE_NAME" + assert_success + bucket_name="$output" + + run setup_bucket "$bucket_name" + assert_success + + run send_rest_go_command_expect_error "403" "AccessDenied" "AWS authentication requires a valid Date or x-amz-date header" "-customDate" "ABCDEFG" + assert_success +} diff --git a/tests/test_xml.sh b/tests/test_xml.sh index 0e363ee0..931d6683 100755 --- a/tests/test_xml.sh +++ b/tests/test_xml.sh @@ -38,11 +38,11 @@ source ./tests/setup_unit.sh run build_xpath_string_for_element "dontcare" "" assert_failure - assert_output -p 'param number 2 is empty' + assert_output -p 'element has no name' run build_xpath_string_for_element "XML With Space" assert_failure - assert_output -p "param 'XML With Space' contains a space" + assert_output -p "element 'XML With Space' contains a space" } @test "get_xml_data - missing params" {