From d0ec284e052892b9992fd98beb2c581293ed84a4 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Thu, 11 Dec 2025 19:21:54 +0400 Subject: [PATCH] feat: adds STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER option in test generation script The `openssl`/`curl` command generator script in `rest_scripts` supports both unsigned streaming payload trailers and signed streaming requests. This update adds support for signed streaming requests with trailers (`STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER`). **Usage** The script generates an OpenSSL command file, which is then used to send the request. Example: ```bash go run tests/rest_scripts/generateCommand.go \ --awsAccessKeyId access \ --awsSecretAccessKey secret \ --client openssl \ --commandType putObject \ --bucketName test \ --payload "hello" \ --payloadType STREAMING-UNSIGNED-PAYLOAD-TRAILER \ --chunkSize 8192 \ --objectKey obj \ --filePath req.txt \ --checksumType crc64nvme ``` You can then send the request with: ```bash openssl s_client -connect 127.0.0.1:7070 -ign_eof < req.txt > response.raw ``` --- .../rest_scripts/command/payloadChunkedAWS.go | 4 +- .../command/payloadStreamingAWSHMACSHA256.go | 97 +++++++++++++++++-- tests/rest_scripts/command/s3Command.go | 17 ++-- tests/rest_scripts/generateCommand.go | 25 ++++- 4 files changed, 127 insertions(+), 16 deletions(-) diff --git a/tests/rest_scripts/command/payloadChunkedAWS.go b/tests/rest_scripts/command/payloadChunkedAWS.go index e3574d4..18780b1 100644 --- a/tests/rest_scripts/command/payloadChunkedAWS.go +++ b/tests/rest_scripts/command/payloadChunkedAWS.go @@ -2,14 +2,16 @@ package command import ( "encoding/hex" - "github.com/versity/versitygw/tests/rest_scripts/logger" "strings" + + "github.com/versity/versitygw/tests/rest_scripts/logger" ) type PayloadChunkedAWS struct { *PayloadChunked serviceString string currentDateTime string + yyyymmdd string lastSignature string emptyByteSignature string signingKey []byte diff --git a/tests/rest_scripts/command/payloadStreamingAWSHMACSHA256.go b/tests/rest_scripts/command/payloadStreamingAWSHMACSHA256.go index aca94d4..a8ec11e 100644 --- a/tests/rest_scripts/command/payloadStreamingAWSHMACSHA256.go +++ b/tests/rest_scripts/command/payloadStreamingAWSHMACSHA256.go @@ -5,22 +5,28 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "hash" "io" "os" ) +const ( + streamPayloadTrailerAlgo = "AWS4-HMAC-SHA256-TRAILER" +) + type PayloadStreamingAWS4HMACSHA256 struct { *PayloadChunkedAWS + hasher hash.Hash } -func NewPayloadStreamingAWS4HMACSHA256(source DataSource, chunkSize int64, serviceString, currentDateTime string) *PayloadStreamingAWS4HMACSHA256 { +func NewPayloadStreamingAWS4HMACSHA256(source DataSource, chunkSize int64, payloadType PayloadType, serviceString string, currentDateTime, yyyymmdd, checksumType string) *PayloadStreamingAWS4HMACSHA256 { return &PayloadStreamingAWS4HMACSHA256{ PayloadChunkedAWS: &PayloadChunkedAWS{ PayloadChunked: &PayloadChunked{ Payload: &Payload{ dataSource: source, - payloadType: StreamingAWS4HMACSHA256Payload, - checksumType: "", + payloadType: payloadType, + checksumType: checksumType, dataSizeCalculated: false, dataSize: 0, }, @@ -28,6 +34,7 @@ func NewPayloadStreamingAWS4HMACSHA256(source DataSource, chunkSize int64, servi }, serviceString: serviceString, currentDateTime: currentDateTime, + yyyymmdd: yyyymmdd, lastSignature: "", emptyByteSignature: SHA256HashZeroBytes, signingKey: nil, @@ -41,7 +48,19 @@ func (s *PayloadStreamingAWS4HMACSHA256) AddInitialSignatureAndSigningKey(initia } func (s *PayloadStreamingAWS4HMACSHA256) GetContentLength() (int64, error) { - return s.getChunkedPayloadContentLength(83, 85) + var trailerSize int64 = 85 + if s.payloadType == StreamingAWS4HMACSHA256PayloadTrailer { + chLength, err := GetBase64ChecksumLength(s.checksumType) + if err != nil { + return 0, err + } + + trailerSize += chLength + 92 + + // sum the checksum length + trailerSize += 16 + int64(len(s.checksumType)) + } + return s.getChunkedPayloadContentLength(83, trailerSize) } func (s *PayloadStreamingAWS4HMACSHA256) addSignature(chunk []byte, outFile *os.File) error { @@ -55,10 +74,70 @@ func (s *PayloadStreamingAWS4HMACSHA256) addSignature(chunk []byte, outFile *os. return nil } +func (s *PayloadStreamingAWS4HMACSHA256) addTrailer(outFile *os.File) error { + checksum, err := s.getBase64Checksum(s.hasher) + if err != nil { + return fmt.Errorf("failed to calculated the trailing checksum: %w", err) + } + tr := fmt.Sprintf("x-amz-checksum-%s", s.checksumType) + trailer := fmt.Sprintf("%s:%s", tr, checksum) + + finalSig := s.calculateTrailerSignature(trailer) + + trailerStr := fmt.Sprintf( + "\r\n%s\r\nx-amz-trailer-signature:%s", + trailer, + finalSig, + ) + + if _, err := outFile.Write([]byte(trailerStr)); err != nil { + return fmt.Errorf("error writing final chunk trailer: %w", err) + } + + s.lastSignature = finalSig + return nil +} + +func (s *PayloadStreamingAWS4HMACSHA256) calculateTrailerSignature(trailer string) string { + trailer += "\n" + strToSign := s.getTrailerChunkStringToSign(trailer) + return hex.EncodeToString(hmacSHA256(s.signingKey, strToSign)) +} + +func (s *PayloadStreamingAWS4HMACSHA256) getTrailerChunkStringToSign(trailer string) string { + hsh := sha256.Sum256([]byte(trailer)) + sig := hex.EncodeToString(hsh[:]) + + prefix := s.getStringToSignPrefix(streamPayloadTrailerAlgo) + + strToSign := fmt.Sprintf("%s\n%s\n%s", + prefix, + s.lastSignature, + sig, + ) + + return strToSign +} + +func (s PayloadStreamingAWS4HMACSHA256) getStringToSignPrefix(algo string) string { + return fmt.Sprintf("%s\n%s\n%s", + algo, + s.currentDateTime, + s.serviceString, + ) +} + func (s *PayloadStreamingAWS4HMACSHA256) getReader() (io.Reader, error) { sourceFile, err := s.dataSource.GetReader() if err != nil { - return nil, fmt.Errorf("error creating tee reader: %w", err) + return nil, fmt.Errorf("error creating reader: %w", err) + } + if s.payloadType == StreamingAWS4HMACSHA256PayloadTrailer && s.checksumType != "" { + s.hasher = s.getChecksumHasher() + sourceFile, err = s.dataSource.GetTeeReader(s.hasher) + if err != nil { + return nil, fmt.Errorf("error creating tee reader: %w", err) + } } return bufio.NewReader(sourceFile), nil } @@ -66,8 +145,12 @@ func (s *PayloadStreamingAWS4HMACSHA256) getReader() (io.Reader, error) { func (s *PayloadStreamingAWS4HMACSHA256) WritePayload(filePath string) error { s.addSignatureFunc = s.addSignature s.getReaderFunc = s.getReader - s.addTrailerFunc = func(outFile *os.File) error { - return nil + if s.payloadType == StreamingAWS4HMACSHA256PayloadTrailer { + s.addTrailerFunc = s.addTrailer + } else { + s.addTrailerFunc = func(outFile *os.File) error { + return nil + } } return s.writeChunkedPayload(filePath) } diff --git a/tests/rest_scripts/command/s3Command.go b/tests/rest_scripts/command/s3Command.go index d5217d6..61c84cc 100644 --- a/tests/rest_scripts/command/s3Command.go +++ b/tests/rest_scripts/command/s3Command.go @@ -7,11 +7,12 @@ import ( "encoding/base64" "encoding/hex" "fmt" - logger "github.com/versity/versitygw/tests/rest_scripts/logger" "os" "sort" "strings" "time" + + logger "github.com/versity/versitygw/tests/rest_scripts/logger" ) const ( @@ -128,10 +129,11 @@ func (s *S3Command) CurlShellCommand() (string, error) { } func (s *S3Command) prepareForBuild() error { + now := time.Now().UTC() if s.IncorrectYearMonthDay { - s.currentDateTime = time.Now().Add(-48 * time.Hour).UTC().Format("20060102T150405Z") + s.currentDateTime = now.Add(-48 * time.Hour).Format("20060102T150405Z") } else { - s.currentDateTime = time.Now().UTC().Format("20060102T150405Z") + s.currentDateTime = now.Format("20060102T150405Z") } protocolAndHost := strings.Split(s.Url, "://") if len(protocolAndHost) != 2 { @@ -184,9 +186,9 @@ func (s *S3Command) preparePayload() error { func (s *S3Command) initializeOpenSSLPayloadAndGetContentLength() error { switch s.PayloadType { - case StreamingAWS4HMACSHA256Payload: + case StreamingAWS4HMACSHA256Payload, StreamingAWS4HMACSHA256PayloadTrailer: serviceString := fmt.Sprintf("%s/%s/%s/aws4_request", s.yearMonthDay, s.AwsRegion, s.ServiceName) - s.payloadOpenSSL = NewPayloadStreamingAWS4HMACSHA256(s.dataSource, int64(s.ChunkSize), serviceString, s.currentDateTime) + s.payloadOpenSSL = NewPayloadStreamingAWS4HMACSHA256(s.dataSource, int64(s.ChunkSize), PayloadType(s.PayloadType), serviceString, s.currentDateTime, s.yearMonthDay, s.ChecksumType) case StreamingUnsignedPayloadTrailer: streamingUnsignedPayloadTrailerImpl := NewStreamingUnsignedPayloadWithTrailer(s.dataSource, int64(s.ChunkSize), s.ChecksumType) streamingUnsignedPayloadTrailerImpl.OmitTrailerOrKey(s.OmitPayloadTrailer, s.OmitPayloadTrailerKey) @@ -214,6 +216,9 @@ func (s *S3Command) addHeaderValues() error { } else { s.headerValues = append(s.headerValues, []string{"host", s.host}) } + if s.PayloadType == StreamingAWS4HMACSHA256PayloadTrailer && s.ChecksumType != "" { + s.headerValues = append(s.headerValues, []string{"x-amz-trailer", fmt.Sprintf("x-amz-checksum-%s", s.ChecksumType)}) + } s.headerValues = append(s.headerValues, []string{"x-amz-content-sha256", s.payloadHash}, []string{"x-amz-date", s.currentDateTime}, @@ -418,7 +423,7 @@ func (s *S3Command) writeOpenSSLPayload(file *os.File) error { awsPayload.AddInitialSignatureAndSigningKey(s.signature, s.signingKey) } switch s.PayloadType { - case UnsignedPayload, "", StreamingUnsignedPayloadTrailer, StreamingAWS4HMACSHA256Payload: + case UnsignedPayload, "", StreamingUnsignedPayloadTrailer, StreamingAWS4HMACSHA256Payload, StreamingAWS4HMACSHA256PayloadTrailer: if err := s.payloadOpenSSL.WritePayload(s.FilePath); err != nil { return fmt.Errorf("error writing payload to openssl file: %w", err) } diff --git a/tests/rest_scripts/generateCommand.go b/tests/rest_scripts/generateCommand.go index 40d8675..94397f1 100644 --- a/tests/rest_scripts/generateCommand.go +++ b/tests/rest_scripts/generateCommand.go @@ -4,10 +4,11 @@ import ( "errors" "flag" "fmt" - "github.com/versity/versitygw/tests/rest_scripts/command" - logger "github.com/versity/versitygw/tests/rest_scripts/logger" "log" "strings" + + "github.com/versity/versitygw/tests/rest_scripts/command" + logger "github.com/versity/versitygw/tests/rest_scripts/logger" ) const ( @@ -271,5 +272,25 @@ func validateConfig() error { if *client == command.CURL && *filePath != "" { return fmt.Errorf("writing to file not currently supported for curl commands") } + if !isValidChecksumType(checksumType) { + return fmt.Errorf("invalid checksum type: %s", *checksumType) + } return nil } + +func isValidChecksumType(input *string) bool { + if input == nil { + return true + } + + // make is case insensitive + chType := strings.ToLower(*input) + checksumType = &chType + + switch chType { + case "", command.ChecksumCRC32, command.ChecksumCRC32C, command.ChecksumCRC64NVME, command.ChecksumSHA1, command.ChecksumSHA256: + return true + default: + return false + } +}