feat: adds integration tests for STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER requests

This commit is contained in:
niksis02
2025-12-26 21:16:01 +04:00
parent 807399459d
commit cc54aad003
5 changed files with 362 additions and 17 deletions
+11 -4
View File
@@ -402,7 +402,7 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
}
sig, err := readBytes(rdr, 64)
if err != nil {
debuglogger.Logf("failed to read '\\r', after chunk signature: %v", err)
debuglogger.Logf("failed to read the chunk signature: %v", err)
return cr.handleRdrErr(err, header)
}
@@ -484,6 +484,10 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
return 0, sig, 0, nil
}
// add the chunk size at the end of header parsing
// to avoid duplication because of header stashing
cr.addChunkSize(chunkSize)
// find the index of chunk ending: '\r\n'
// skip the first 2 bytes as it is the starting '\r\n'
// the first chunk doesn't contain the starting '\r\n', but
@@ -511,7 +515,7 @@ func (cr *ChunkReader) stashAndSkipHeader(header []byte) (int64, string, int, er
// calls "cr.stashAndSkipHeader" if the passed err is "io.EOF" and cr.isEOF is false
// Returns the error otherwise
func (cr *ChunkReader) handleRdrErr(err error, header []byte) (int64, string, int, error) {
if err == io.EOF {
if err == io.EOF || err == io.ErrUnexpectedEOF {
if cr.isEOF {
debuglogger.Logf("incomplete chunk encoding, EOF reached")
return 0, "", 0, s3err.GetAPIError(s3err.ErrIncompleteBody)
@@ -540,11 +544,14 @@ func (cr *ChunkReader) parseChunkSize(rdr *bufio.Reader, header []byte) (int64,
return 0, s3err.GetAPIError(s3err.ErrInvalidChunkSize)
}
cr.chunkSizes = append(cr.chunkSizes, chunkSize)
return chunkSize, nil
}
// addChunkSize adds the input chunk size to chunkSizes slice
func (cr *ChunkReader) addChunkSize(size int64) {
cr.chunkSizes = append(cr.chunkSizes, size)
}
// isValidChunkSize checks if the parsed chunk size is valid
// they follow one rule: all chunk sizes except for the last one
// should be greater than 8192
+19
View File
@@ -808,6 +808,8 @@ func TestFullFlow(ts *TestState) {
TestAccessControl(ts)
TestRouter(ts)
TestUnsignedStreaminPayloadTrailer(ts)
TestSignedStreaminPayload(ts)
TestSignedStreaminPayloadTrailer(ts)
// FIXME: The tests should pass for azure as well
// but this issue should be fixed with https://github.com/versity/versitygw/issues/1336
if !ts.conf.azureTests {
@@ -1122,6 +1124,17 @@ func TestSignedStreaminPayload(ts *TestState) {
}
}
func TestSignedStreaminPayloadTrailer(ts *TestState) {
if !ts.conf.azureTests {
ts.Run(SignedStreamingPayloadTrailer_malformed_trailer)
ts.Run(SignedStreamingPayloadTrailer_incomplete_body)
ts.Run(SignedStreamingPayloadTrailer_missing_x_amz_trailer_header)
ts.Run(SignedStreamingPayloadTrailer_invalid_checksum)
ts.Run(SignedStreamingPayloadTrailer_bad_digest)
ts.Run(SignedStreamingPayloadTrailer_success)
}
}
type IntTest func(s3 *S3Conf) error
type IntTests map[string]IntTest
@@ -1778,5 +1791,11 @@ func GetIntTests() IntTests {
"SignedStreamingPayload_invalid_encoding": SignedStreamingPayload_invalid_encoding,
"SignedStreamingPayload_invalid_chunk_size": SignedStreamingPayload_invalid_chunk_size,
"SignedStreamingPayload_decoded_content_length_mismatch": SignedStreamingPayload_decoded_content_length_mismatch,
"SignedStreamingPayloadTrailer_malformed_trailer": SignedStreamingPayloadTrailer_malformed_trailer,
"SignedStreamingPayloadTrailer_incomplete_body": SignedStreamingPayloadTrailer_incomplete_body,
"SignedStreamingPayloadTrailer_missing_x_amz_trailer_header": SignedStreamingPayloadTrailer_missing_x_amz_trailer_header,
"SignedStreamingPayloadTrailer_invalid_checksum": SignedStreamingPayloadTrailer_invalid_checksum,
"SignedStreamingPayloadTrailer_bad_digest": SignedStreamingPayloadTrailer_bad_digest,
"SignedStreamingPayloadTrailer_success": SignedStreamingPayloadTrailer_success,
}
}
@@ -74,9 +74,13 @@ func SignedStreamingPayload_invalid_chunk_size(s *S3Conf) error {
{10, bytes.Repeat([]byte{'b'}, 100), true},
{1000, bytes.Repeat([]byte{'a'}, 200), false},
{8192, bytes.Repeat([]byte{'c'}, 10000), false},
{8192, bytes.Repeat([]byte{'c'}, 20000), false},
{1000, bytes.Repeat([]byte{'c'}, 1024*64), true},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, test.payload, withChunkSize(test.chunkSize))
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, test.payload, withChunkSize(test.chunkSize), withCustomHeaders(map[string]string{
"Content-Length": "-1",
"Transfer-Encoding": "chunked",
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
@@ -0,0 +1,207 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package integration
import (
"fmt"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
)
func SignedStreamingPayloadTrailer_malformed_trailer(s *S3Conf) error {
testName := "SignedStreamingPayloadTrailer_malformed_trailer"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
trailerHdr string
trailingChecksum string
}{
{"x-amz-checksum-crc64nvme", "x-amz-invalid:invalid"},
{"x-amz-checksum-crc64nvme", ""},
// x-amz-trailer and trailing checksum mismatch
{"x-amz-checksum-sha1", "x-amz-checksum-crc32:QWaN2w=="},
{"x-amz-checksum-crc32c", "x-amz-checksum-sha1:YR/1TvTYOJz5gtqVFoBJBtmTibY="},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": test.trailerHdr,
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
if err := compareS3ApiError(s3err.GetAPIError(s3err.ErrMalformedTrailer), apiErr); err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
}
return nil
})
}
func SignedStreamingPayloadTrailer_incomplete_body(s *S3Conf) error {
testName := "SignedStreamingPayloadTrailer_incomplete_body"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
modifFrom int
modifTo int
modifPayload []byte
}{
{175, 176, []byte("k")},
{175, 177, []byte("cc")},
{215, 216, []byte("bcd")},
{220, 223, []byte("invalid")},
{230, 235, []byte("abcd")},
{241, 245, []byte("abcde")},
{306, 308, []byte("pp")},
{304, 308, []byte("erty")},
} {
_, apiErr, err := testSignedStreamingObjectPut(
s,
bucket,
object,
[]byte("abcdefg"),
withTrailingChecksum("x-amz-checksum-crc64nvme:SmzZ/LTp1CA="),
withCustomHeaders(map[string]string{"x-amz-trailer": "x-amz-checksum-crc64nvme"}),
withModifyPayload(test.modifFrom, test.modifTo, test.modifPayload),
)
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
if err := compareS3ApiError(s3err.GetAPIError(s3err.ErrIncompleteBody), apiErr); err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
}
return nil
})
}
func SignedStreamingPayloadTrailer_missing_x_amz_trailer_header(s *S3Conf) error {
testName := "SignedStreamingPayloadTrailer_missing_x_amz_trailer_header"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, "my-object", []byte("hello"), withTrailingChecksum("x-amz-checksum-crc32:NhCmhg=="))
if err != nil {
return err
}
return compareS3ApiError(s3err.GetAPIError(s3err.ErrMalformedTrailer), apiErr)
})
}
func SignedStreamingPayloadTrailer_invalid_checksum(s *S3Conf) error {
testName := "SignedStreamingPayloadTrailer_invalid_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
trailerHdr string
trailingChecksum string
}{
{"x-amz-checksum-crc32", "x-amz-checksum-crc32:invalid"},
{"x-amz-checksum-crc32c", "x-amz-checksum-crc32c:invalid"},
{"x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:invalid"},
{"x-amz-checksum-sha1", "x-amz-checksum-sha1:invalid"},
{"x-amz-checksum-sha256", "x-amz-checksum-sha256:invalid"},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": test.trailerHdr,
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(test.trailerHdr), apiErr); err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
}
return nil
})
}
func SignedStreamingPayloadTrailer_bad_digest(s *S3Conf) error {
testName := "SignedStreamingPayloadTrailer_bad_digest"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
algo types.ChecksumAlgorithm
trailerHdr string
trailingChecksum string
}{
{types.ChecksumAlgorithmCrc32, "x-amz-checksum-crc32", "x-amz-checksum-crc32:NhCmhg=="},
{types.ChecksumAlgorithmCrc32c, "x-amz-checksum-crc32c", "x-amz-checksum-crc32c:+Cy97w=="},
{types.ChecksumAlgorithmCrc64nvme, "x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:QFRKMGE3tuw="},
{types.ChecksumAlgorithmSha1, "x-amz-checksum-sha1", "x-amz-checksum-sha1:qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{types.ChecksumAlgorithmSha256, "x-amz-checksum-sha256", "x-amz-checksum-sha256:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": test.trailerHdr,
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
if err := compareS3ApiError(s3err.GetChecksumBadDigestErr(test.algo), apiErr); err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
}
return nil
})
}
func SignedStreamingPayloadTrailer_success(s *S3Conf) error {
testName := "SignedStreamingPayloadTrailer_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
checksumKey string
checksumValue string
}{
{"x-amz-checksum-crc32", "z3mWAA=="},
{"x-amz-checksum-crc32c", "rxvjPA=="},
{"x-amz-checksum-crc64nvme", "dYnI3/Fh0gM="},
{"x-amz-checksum-sha1", "8O8FwCfmd5fCbCBvH09mrKMVoHU="},
{"x-amz-checksum-sha256", "OoSow5X4zTIPl27MtdFdYT+9O3C367C75+Cb2MFtRBc="},
} {
headers, apiErr, err := testSignedStreamingObjectPut(
s,
bucket,
object,
[]byte("the object data"),
withTrailingChecksum(fmt.Sprintf("%s:%s", test.checksumKey, test.checksumValue)),
withCustomHeaders(map[string]string{
"x-amz-trailer": test.checksumKey,
}),
)
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
if apiErr != nil {
return fmt.Errorf("test %v failed: (%s) %s", i+1, apiErr.Code, apiErr.Message)
}
if headers[test.checksumKey] != test.checksumValue {
return fmt.Errorf("test %v failed: expected %s header value to be %s, instead got %s", i+1, test.checksumKey, test.checksumValue, headers[test.checksumKey])
}
}
return nil
})
}
+120 -12
View File
@@ -17,6 +17,7 @@ package integration
import (
"bytes"
"context"
"crypto/hmac"
"crypto/md5"
"crypto/rand"
"crypto/sha1"
@@ -2120,11 +2121,13 @@ func constructUnsignedPaylod(chunkSizes ...int64) (int64, []byte, error) {
}
type signedReqCfg struct {
headers map[string]string
chunkSize int64
modifFrom *int
modifTo *int
modifPayload []byte
headers map[string]string
chunkSize int64
modifFrom *int
modifTo *int
modifPayload []byte
trailingChecksum *string
isTrailer bool
}
type signedReqOpt func(*signedReqCfg)
@@ -2145,6 +2148,13 @@ func withModifyPayload(from int, to int, p []byte) signedReqOpt {
}
}
func withTrailingChecksum(checksum string) signedReqOpt {
return func(src *signedReqCfg) {
src.trailingChecksum = &checksum
src.isTrailer = true
}
}
func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []byte, opts ...signedReqOpt) (map[string]string, *s3err.APIErrorResponse, error) {
cfg := &signedReqCfg{
chunkSize: 8192, // minimal valid chunk size
@@ -2162,6 +2172,7 @@ func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []by
}
var payloadOffset int64
var trailerLength int
// any planned modification which is going to affect the
// Content-Length header value
@@ -2169,10 +2180,17 @@ func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []by
diff := len(cfg.modifPayload) - *cfg.modifTo + *cfg.modifFrom
payloadOffset = int64(diff)
}
if cfg.isTrailer {
trailerLength = len(*cfg.trailingChecksum)
}
// precalculated the Content-Length header to correctly sign the request
req.ContentLength = calculateSignedReqContentLength(int64(len(payload)), cfg.chunkSize, payloadOffset)
req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
req.ContentLength = calculateSignedReqContentLength(int64(len(payload)), cfg.chunkSize, payloadOffset, cfg.isTrailer, int64(trailerLength))
sha256Header := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
if cfg.isTrailer {
sha256Header = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
}
req.Header.Set("x-amz-decoded-content-length", fmt.Sprint(len(payload)))
req.Header.Set("x-amz-content-sha256", sha256Header)
// set custom request headers
for key, val := range cfg.headers {
@@ -2183,7 +2201,7 @@ func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []by
signingTime := time.Now()
// sign the request
err = signer.SignHTTP(ctx, aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, req, "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", "s3", s.awsRegion, signingTime)
err = signer.SignHTTP(ctx, aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, req, sha256Header, "s3", s.awsRegion, signingTime)
if err != nil {
return nil, nil, cancelAndError(fmt.Errorf("failed to sign the request: %w", err), cancel)
}
@@ -2197,7 +2215,7 @@ func testSignedStreamingObjectPut(s *S3Conf, bucket, object string, payload []by
// initialize v4 stream signed
streamSigner := v4.NewStreamSigner(aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, "s3", s.awsRegion, seedSignature)
// create the signed payload
body, err := constructSignedStreamingPayload(ctx, streamSigner, signingTime, payload, cfg.chunkSize)
body, err := constructSignedStreamingPayload(ctx, streamSigner, signingTime, payload, cfg.chunkSize, cfg.trailingChecksum, s.awsRegion, s.awsSecret)
if err != nil {
return nil, nil, cancelAndError(fmt.Errorf("failed to encode req body: %w", err), cancel)
}
@@ -2254,18 +2272,23 @@ func cancelAndError(err error, cancel context.CancelFunc) error {
const (
chunkSigHdrLength int64 = 81
trailerSigLength int64 = 88
)
// calculateSignedReqContentLength calculates the value of `Content-Length` header
// sizeOffset marks any planned changes on the body, which will affect the size
func calculateSignedReqContentLength(decPayloadSize int64, chunkSize int64, sizeOffset int64) int64 {
func calculateSignedReqContentLength(decPayloadSize int64, chunkSize int64, sizeOffset int64, withTrailer bool, trailerLength int64) int64 {
payloadSize := decPayloadSize
var chunkHeadersLength int64
if withTrailer {
chunkHeadersLength += trailerLength + 4 + trailerSigLength
}
// special case when chunk size is greater or equal than decoded content length
if chunkSize >= decPayloadSize {
chSizeLgth := len(fmt.Sprintf("%x", decPayloadSize))
return decPayloadSize + sizeOffset + int64(chSizeLgth) + 2*chunkSigHdrLength + 9
return decPayloadSize + sizeOffset + int64(chSizeLgth) + 2*chunkSigHdrLength + 9 + chunkHeadersLength
}
for {
@@ -2287,7 +2310,7 @@ func calculateSignedReqContentLength(decPayloadSize int64, chunkSize int64, size
}
// constructSignedStreamingPayload creates chunk encoded payload with signatures.
func constructSignedStreamingPayload(ctx context.Context, signer *v4.StreamSigner, signingTime time.Time, payload []byte, chunkSize int64) ([]byte, error) {
func constructSignedStreamingPayload(ctx context.Context, signer *v4.StreamSigner, signingTime time.Time, payload []byte, chunkSize int64, trailer *string, region, secret string) ([]byte, error) {
buf := bytes.NewBuffer(nil)
payloadLen := int64(len(payload))
@@ -2326,6 +2349,26 @@ func constructSignedStreamingPayload(ctx context.Context, signer *v4.StreamSigne
return nil, err
}
if trailer != nil {
_, err = buf.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n", sig))
if err != nil {
return nil, err
}
sigKey := getSigningKey(secret, signingTime.Format("20060102"), region)
trailerSig, err := getAWS4StreamingTrailer(sigKey, sig, signingTime, region, *trailer)
if err != nil {
return nil, err
}
_, err = buf.WriteString(fmt.Sprintf("%s\r\nx-amz-trailer-signature:%s\r\n\r\n", *trailer, trailerSig))
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
_, err = buf.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n\r\n", sig))
if err != nil {
return nil, err
@@ -2379,3 +2422,68 @@ func replaceRange(dst, src []byte, start, end int) ([]byte, error) {
copy(out[start+len(src):], dst[end:])
return out, nil
}
func getAWS4StreamingTrailer(
signingKey,
lastSignature []byte,
signingTime time.Time,
awsRegion,
trailer string,
) (string, error) {
// yyyyMMdd
yearMonthDay := signingTime.Format("20060102")
// ISO8601 basic format: yyyyMMdd'T'HHmmss'Z'
currentDateTime := signingTime.UTC().Format("20060102T150405Z")
// <date>/<region>/<service>/aws4_request
serviceString := fmt.Sprintf(
"%s/%s/s3/aws4_request",
yearMonthDay,
awsRegion,
)
// Trailer must be newline-terminated for hashing/signing
trailerWithNL := trailer + "\n"
// Hash of trailer
trailerHash := sha256.Sum256([]byte(trailerWithNL))
trailerHashHex := hex.EncodeToString(trailerHash[:])
// String-to-sign prefix
stringToSignPrefix := fmt.Sprintf(
"%s\n%s\n%s",
"AWS4-HMAC-SHA256-TRAILER",
currentDateTime,
serviceString,
)
// Full string-to-sign
stringToSign := fmt.Sprintf(
"%s\n%x\n%s",
stringToSignPrefix,
lastSignature,
trailerHashHex,
)
// Final trailer signature
finalSignature := hex.EncodeToString(
hmacSHA256(signingKey, stringToSign),
)
return finalSignature, nil
}
func hmacSHA256(key []byte, data string) []byte {
h := hmac.New(sha256.New, key)
h.Write([]byte(data))
return h.Sum(nil)
}
func getSigningKey(secret, yearMonthDay, region string) []byte {
dateKey := hmacSHA256([]byte("AWS4"+secret), yearMonthDay)
dateRegionKey := hmacSHA256(dateKey, region)
dateRegionServiceKey := hmacSHA256(dateRegionKey, "s3")
return hmacSHA256(dateRegionServiceKey, "aws4_request")
}