diff --git a/s3api/utils/signed-chunk-reader.go b/s3api/utils/signed-chunk-reader.go index ed43881d..f194d652 100644 --- a/s3api/utils/signed-chunk-reader.go +++ b/s3api/utils/signed-chunk-reader.go @@ -402,7 +402,7 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, } sig, err := readBytes(rdr, 64) if err != nil { - debuglogger.Logf("failed to read '\\r', after chunk signature: %v", err) + debuglogger.Logf("failed to read the chunk signature: %v", err) return cr.handleRdrErr(err, header) } @@ -484,6 +484,10 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, return 0, sig, 0, nil } + // add the chunk size at the end of header parsing + // to avoid duplication because of header stashing + cr.addChunkSize(chunkSize) + // find the index of chunk ending: '\r\n' // skip the first 2 bytes as it is the starting '\r\n' // the first chunk doesn't contain the starting '\r\n', but @@ -511,7 +515,7 @@ func (cr *ChunkReader) stashAndSkipHeader(header []byte) (int64, string, int, er // calls "cr.stashAndSkipHeader" if the passed err is "io.EOF" and cr.isEOF is false // Returns the error otherwise func (cr *ChunkReader) handleRdrErr(err error, header []byte) (int64, string, int, error) { - if err == io.EOF { + if err == io.EOF || err == io.ErrUnexpectedEOF { if cr.isEOF { debuglogger.Logf("incomplete chunk encoding, EOF reached") return 0, "", 0, s3err.GetAPIError(s3err.ErrIncompleteBody) @@ -540,11 +544,14 @@ func (cr *ChunkReader) parseChunkSize(rdr *bufio.Reader, header []byte) (int64, return 0, s3err.GetAPIError(s3err.ErrInvalidChunkSize) } - cr.chunkSizes = append(cr.chunkSizes, chunkSize) - return chunkSize, nil } +// addChunkSize adds the input chunk size to chunkSizes slice +func (cr *ChunkReader) addChunkSize(size int64) { + cr.chunkSizes = append(cr.chunkSizes, size) +} + // isValidChunkSize checks if the parsed chunk size is valid // they follow one rule: all chunk sizes except for the last one // should be greater than 8192 diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 863a9bbd..12a92f7d 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -808,6 +808,8 @@ func TestFullFlow(ts *TestState) { TestAccessControl(ts) TestRouter(ts) TestUnsignedStreaminPayloadTrailer(ts) + TestSignedStreaminPayload(ts) + TestSignedStreaminPayloadTrailer(ts) // FIXME: The tests should pass for azure as well // but this issue should be fixed with https://github.com/versity/versitygw/issues/1336 if !ts.conf.azureTests { @@ -1122,6 +1124,17 @@ func TestSignedStreaminPayload(ts *TestState) { } } +func TestSignedStreaminPayloadTrailer(ts *TestState) { + if !ts.conf.azureTests { + ts.Run(SignedStreamingPayloadTrailer_malformed_trailer) + ts.Run(SignedStreamingPayloadTrailer_incomplete_body) + ts.Run(SignedStreamingPayloadTrailer_missing_x_amz_trailer_header) + ts.Run(SignedStreamingPayloadTrailer_invalid_checksum) + ts.Run(SignedStreamingPayloadTrailer_bad_digest) + ts.Run(SignedStreamingPayloadTrailer_success) + } +} + type IntTest func(s3 *S3Conf) error type IntTests map[string]IntTest @@ -1778,5 +1791,11 @@ func GetIntTests() IntTests { "SignedStreamingPayload_invalid_encoding": SignedStreamingPayload_invalid_encoding, "SignedStreamingPayload_invalid_chunk_size": SignedStreamingPayload_invalid_chunk_size, "SignedStreamingPayload_decoded_content_length_mismatch": SignedStreamingPayload_decoded_content_length_mismatch, + "SignedStreamingPayloadTrailer_malformed_trailer": SignedStreamingPayloadTrailer_malformed_trailer, + "SignedStreamingPayloadTrailer_incomplete_body": SignedStreamingPayloadTrailer_incomplete_body, + "SignedStreamingPayloadTrailer_missing_x_amz_trailer_header": SignedStreamingPayloadTrailer_missing_x_amz_trailer_header, + "SignedStreamingPayloadTrailer_invalid_checksum": SignedStreamingPayloadTrailer_invalid_checksum, + "SignedStreamingPayloadTrailer_bad_digest": SignedStreamingPayloadTrailer_bad_digest, + "SignedStreamingPayloadTrailer_success": SignedStreamingPayloadTrailer_success, } } diff --git a/tests/integration/signed-streaming-payload.go b/tests/integration/signed_streaming_payload.go similarity index 94% rename from tests/integration/signed-streaming-payload.go rename to tests/integration/signed_streaming_payload.go index 4e461efc..35caad8f 100644 --- a/tests/integration/signed-streaming-payload.go +++ b/tests/integration/signed_streaming_payload.go @@ -74,9 +74,13 @@ func SignedStreamingPayload_invalid_chunk_size(s *S3Conf) error { {10, bytes.Repeat([]byte{'b'}, 100), true}, {1000, bytes.Repeat([]byte{'a'}, 200), false}, {8192, bytes.Repeat([]byte{'c'}, 10000), false}, + {8192, bytes.Repeat([]byte{'c'}, 20000), false}, {1000, bytes.Repeat([]byte{'c'}, 1024*64), true}, } { - _, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, test.payload, withChunkSize(test.chunkSize)) + _, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, test.payload, withChunkSize(test.chunkSize), withCustomHeaders(map[string]string{ + "Content-Length": "-1", + "Transfer-Encoding": "chunked", + })) if err != nil { return fmt.Errorf("test %v failed: %w", i+1, err) } diff --git a/tests/integration/signed_streaming_payload_trailer.go b/tests/integration/signed_streaming_payload_trailer.go new file mode 100644 index 00000000..5a16aa83 --- /dev/null +++ b/tests/integration/signed_streaming_payload_trailer.go @@ -0,0 +1,207 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "fmt" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +func SignedStreamingPayloadTrailer_malformed_trailer(s *S3Conf) error { + testName := "SignedStreamingPayloadTrailer_malformed_trailer" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + object := "my-object" + for i, test := range []struct { + trailerHdr string + trailingChecksum string + }{ + {"x-amz-checksum-crc64nvme", "x-amz-invalid:invalid"}, + {"x-amz-checksum-crc64nvme", ""}, + // x-amz-trailer and trailing checksum mismatch + {"x-amz-checksum-sha1", "x-amz-checksum-crc32:QWaN2w=="}, + {"x-amz-checksum-crc32c", "x-amz-checksum-sha1:YR/1TvTYOJz5gtqVFoBJBtmTibY="}, + } { + _, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{ + "x-amz-trailer": test.trailerHdr, + })) + if err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + + if err := compareS3ApiError(s3err.GetAPIError(s3err.ErrMalformedTrailer), apiErr); err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + } + + return nil + }) +} + +func SignedStreamingPayloadTrailer_incomplete_body(s *S3Conf) error { + testName := "SignedStreamingPayloadTrailer_incomplete_body" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + object := "my-object" + for i, test := range []struct { + modifFrom int + modifTo int + modifPayload []byte + }{ + {175, 176, []byte("k")}, + {175, 177, []byte("cc")}, + {215, 216, []byte("bcd")}, + {220, 223, []byte("invalid")}, + {230, 235, []byte("abcd")}, + {241, 245, []byte("abcde")}, + {306, 308, []byte("pp")}, + {304, 308, []byte("erty")}, + } { + _, apiErr, err := testSignedStreamingObjectPut( + s, + bucket, + object, + []byte("abcdefg"), + withTrailingChecksum("x-amz-checksum-crc64nvme:SmzZ/LTp1CA="), + withCustomHeaders(map[string]string{"x-amz-trailer": "x-amz-checksum-crc64nvme"}), + withModifyPayload(test.modifFrom, test.modifTo, test.modifPayload), + ) + if err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + + if err := compareS3ApiError(s3err.GetAPIError(s3err.ErrIncompleteBody), apiErr); err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + } + + return nil + }) +} + +func SignedStreamingPayloadTrailer_missing_x_amz_trailer_header(s *S3Conf) error { + testName := "SignedStreamingPayloadTrailer_missing_x_amz_trailer_header" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + _, apiErr, err := testSignedStreamingObjectPut(s, bucket, "my-object", []byte("hello"), withTrailingChecksum("x-amz-checksum-crc32:NhCmhg==")) + if err != nil { + return err + } + + return compareS3ApiError(s3err.GetAPIError(s3err.ErrMalformedTrailer), apiErr) + }) +} + +func SignedStreamingPayloadTrailer_invalid_checksum(s *S3Conf) error { + testName := "SignedStreamingPayloadTrailer_invalid_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + object := "my-object" + for i, test := range []struct { + trailerHdr string + trailingChecksum string + }{ + {"x-amz-checksum-crc32", "x-amz-checksum-crc32:invalid"}, + {"x-amz-checksum-crc32c", "x-amz-checksum-crc32c:invalid"}, + {"x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:invalid"}, + {"x-amz-checksum-sha1", "x-amz-checksum-sha1:invalid"}, + {"x-amz-checksum-sha256", "x-amz-checksum-sha256:invalid"}, + } { + _, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{ + "x-amz-trailer": test.trailerHdr, + })) + if err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + + if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(test.trailerHdr), apiErr); err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + } + + return nil + }) +} + +func SignedStreamingPayloadTrailer_bad_digest(s *S3Conf) error { + testName := "SignedStreamingPayloadTrailer_bad_digest" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + object := "my-object" + for i, test := range []struct { + algo types.ChecksumAlgorithm + trailerHdr string + trailingChecksum string + }{ + {types.ChecksumAlgorithmCrc32, "x-amz-checksum-crc32", "x-amz-checksum-crc32:NhCmhg=="}, + {types.ChecksumAlgorithmCrc32c, "x-amz-checksum-crc32c", "x-amz-checksum-crc32c:+Cy97w=="}, + {types.ChecksumAlgorithmCrc64nvme, "x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:QFRKMGE3tuw="}, + {types.ChecksumAlgorithmSha1, "x-amz-checksum-sha1", "x-amz-checksum-sha1:qvTGHdzF6KLavt4PO0gs2a6pQ00="}, + {types.ChecksumAlgorithmSha256, "x-amz-checksum-sha256", "x-amz-checksum-sha256:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="}, + } { + _, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{ + "x-amz-trailer": test.trailerHdr, + })) + if err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + + if err := compareS3ApiError(s3err.GetChecksumBadDigestErr(test.algo), apiErr); err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + } + + return nil + }) +} + +func SignedStreamingPayloadTrailer_success(s *S3Conf) error { + testName := "SignedStreamingPayloadTrailer_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + object := "my-object" + for i, test := range []struct { + checksumKey string + checksumValue string + }{ + {"x-amz-checksum-crc32", "z3mWAA=="}, + {"x-amz-checksum-crc32c", "rxvjPA=="}, + {"x-amz-checksum-crc64nvme", "dYnI3/Fh0gM="}, + {"x-amz-checksum-sha1", "8O8FwCfmd5fCbCBvH09mrKMVoHU="}, + {"x-amz-checksum-sha256", "OoSow5X4zTIPl27MtdFdYT+9O3C367C75+Cb2MFtRBc="}, + } { + headers, apiErr, err := testSignedStreamingObjectPut( + s, + bucket, + object, + []byte("the object data"), + withTrailingChecksum(fmt.Sprintf("%s:%s", test.checksumKey, test.checksumValue)), + withCustomHeaders(map[string]string{ + "x-amz-trailer": test.checksumKey, + }), + ) + + if err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + if apiErr != nil { + return fmt.Errorf("test %v failed: (%s) %s", i+1, apiErr.Code, apiErr.Message) + } + + if headers[test.checksumKey] != test.checksumValue { + return fmt.Errorf("test %v failed: expected %s header value to be %s, instead got %s", i+1, test.checksumKey, test.checksumValue, headers[test.checksumKey]) + } + } + + return nil + }) +} diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 10342d42..9802b046 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -17,6 +17,7 @@ package integration import ( "bytes" "context" + "crypto/hmac" "crypto/md5" "crypto/rand" "crypto/sha1" @@ -2120,11 +2121,13 @@ func constructUnsignedPaylod(chunkSizes ...int64) (int64, []byte, error) { } type signedReqCfg struct { - headers map[string]string - chunkSize int64 - modifFrom *int - modifTo *int - modifPayload []byte + headers map[string]string + chunkSize int64 + modifFrom *int + modifTo *int + modifPayload []byte + trailingChecksum *string + isTrailer bool } type signedReqOpt func(*signedReqCfg) @@ -2145,6 +2148,13 @@ func withModifyPayload(from int, to int, p []byte) signedReqOpt { } } +func withTrailingChecksum(checksum string) signedReqOpt { + return func(src *signedReqCfg) { + src.trailingChecksum = &checksum + src.isTrailer = true + } +} + func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []byte, opts ...signedReqOpt) (map[string]string, *s3err.APIErrorResponse, error) { cfg := &signedReqCfg{ chunkSize: 8192, // minimal valid chunk size @@ -2162,6 +2172,7 @@ func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []by } var payloadOffset int64 + var trailerLength int // any planned modification which is going to affect the // Content-Length header value @@ -2169,10 +2180,17 @@ func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []by diff := len(cfg.modifPayload) - *cfg.modifTo + *cfg.modifFrom payloadOffset = int64(diff) } + if cfg.isTrailer { + trailerLength = len(*cfg.trailingChecksum) + } // precalculated the Content-Length header to correctly sign the request - req.ContentLength = calculateSignedReqContentLength(int64(len(payload)), cfg.chunkSize, payloadOffset) - req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD") + req.ContentLength = calculateSignedReqContentLength(int64(len(payload)), cfg.chunkSize, payloadOffset, cfg.isTrailer, int64(trailerLength)) + sha256Header := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + if cfg.isTrailer { + sha256Header = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + } req.Header.Set("x-amz-decoded-content-length", fmt.Sprint(len(payload))) + req.Header.Set("x-amz-content-sha256", sha256Header) // set custom request headers for key, val := range cfg.headers { @@ -2183,7 +2201,7 @@ func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []by signingTime := time.Now() // sign the request - err = signer.SignHTTP(ctx, aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, req, "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", "s3", s.awsRegion, signingTime) + err = signer.SignHTTP(ctx, aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, req, sha256Header, "s3", s.awsRegion, signingTime) if err != nil { return nil, nil, cancelAndError(fmt.Errorf("failed to sign the request: %w", err), cancel) } @@ -2197,7 +2215,7 @@ func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []by // initialize v4 stream signed streamSigner := v4.NewStreamSigner(aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, "s3", s.awsRegion, seedSignature) // create the signed payload - body, err := constructSignedStreamingPayload(ctx, streamSigner, signingTime, payload, cfg.chunkSize) + body, err := constructSignedStreamingPayload(ctx, streamSigner, signingTime, payload, cfg.chunkSize, cfg.trailingChecksum, s.awsRegion, s.awsSecret) if err != nil { return nil, nil, cancelAndError(fmt.Errorf("failed to encode req body: %w", err), cancel) } @@ -2254,18 +2272,23 @@ func cancelAndError(err error, cancel context.CancelFunc) error { const ( chunkSigHdrLength int64 = 81 + trailerSigLength int64 = 88 ) // calculateSignedReqContentLength calculates the value of `Content-Length` header // sizeOffset marks any planned changes on the body, which will affect the size -func calculateSignedReqContentLength(decPayloadSize int64, chunkSize int64, sizeOffset int64) int64 { +func calculateSignedReqContentLength(decPayloadSize int64, chunkSize int64, sizeOffset int64, withTrailer bool, trailerLength int64) int64 { payloadSize := decPayloadSize var chunkHeadersLength int64 + if withTrailer { + chunkHeadersLength += trailerLength + 4 + trailerSigLength + } + // special case when chunk size is greater or equal than decoded content length if chunkSize >= decPayloadSize { chSizeLgth := len(fmt.Sprintf("%x", decPayloadSize)) - return decPayloadSize + sizeOffset + int64(chSizeLgth) + 2*chunkSigHdrLength + 9 + return decPayloadSize + sizeOffset + int64(chSizeLgth) + 2*chunkSigHdrLength + 9 + chunkHeadersLength } for { @@ -2287,7 +2310,7 @@ func calculateSignedReqContentLength(decPayloadSize int64, chunkSize int64, size } // constructSignedStreamingPayload creates chunk encoded payload with signatures. -func constructSignedStreamingPayload(ctx context.Context, signer *v4.StreamSigner, signingTime time.Time, payload []byte, chunkSize int64) ([]byte, error) { +func constructSignedStreamingPayload(ctx context.Context, signer *v4.StreamSigner, signingTime time.Time, payload []byte, chunkSize int64, trailer *string, region, secret string) ([]byte, error) { buf := bytes.NewBuffer(nil) payloadLen := int64(len(payload)) @@ -2326,6 +2349,26 @@ func constructSignedStreamingPayload(ctx context.Context, signer *v4.StreamSigne return nil, err } + if trailer != nil { + _, err = buf.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n", sig)) + if err != nil { + return nil, err + } + + sigKey := getSigningKey(secret, signingTime.Format("20060102"), region) + trailerSig, err := getAWS4StreamingTrailer(sigKey, sig, signingTime, region, *trailer) + if err != nil { + return nil, err + } + + _, err = buf.WriteString(fmt.Sprintf("%s\r\nx-amz-trailer-signature:%s\r\n\r\n", *trailer, trailerSig)) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil + } + _, err = buf.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n\r\n", sig)) if err != nil { return nil, err @@ -2379,3 +2422,68 @@ func replaceRange(dst, src []byte, start, end int) ([]byte, error) { copy(out[start+len(src):], dst[end:]) return out, nil } + +func getAWS4StreamingTrailer( + signingKey, + lastSignature []byte, + signingTime time.Time, + awsRegion, + trailer string, +) (string, error) { + + // yyyyMMdd + yearMonthDay := signingTime.Format("20060102") + + // ISO8601 basic format: yyyyMMdd'T'HHmmss'Z' + currentDateTime := signingTime.UTC().Format("20060102T150405Z") + + // ///aws4_request + serviceString := fmt.Sprintf( + "%s/%s/s3/aws4_request", + yearMonthDay, + awsRegion, + ) + + // Trailer must be newline-terminated for hashing/signing + trailerWithNL := trailer + "\n" + + // Hash of trailer + trailerHash := sha256.Sum256([]byte(trailerWithNL)) + trailerHashHex := hex.EncodeToString(trailerHash[:]) + + // String-to-sign prefix + stringToSignPrefix := fmt.Sprintf( + "%s\n%s\n%s", + "AWS4-HMAC-SHA256-TRAILER", + currentDateTime, + serviceString, + ) + + // Full string-to-sign + stringToSign := fmt.Sprintf( + "%s\n%x\n%s", + stringToSignPrefix, + lastSignature, + trailerHashHex, + ) + + // Final trailer signature + finalSignature := hex.EncodeToString( + hmacSHA256(signingKey, stringToSign), + ) + + return finalSignature, nil +} + +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + h.Write([]byte(data)) + return h.Sum(nil) +} + +func getSigningKey(secret, yearMonthDay, region string) []byte { + dateKey := hmacSHA256([]byte("AWS4"+secret), yearMonthDay) + dateRegionKey := hmacSHA256(dateKey, region) + dateRegionServiceKey := hmacSHA256(dateRegionKey, "s3") + return hmacSHA256(dateRegionServiceKey, "aws4_request") +}