feat: adds STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER option in test generation script

The `openssl`/`curl` command generator script in `rest_scripts` supports both unsigned streaming payload trailers and signed streaming requests. This update adds support for signed streaming requests with trailers (`STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER`).

**Usage**
The script generates an OpenSSL command file, which is then used to send the request.

Example:

```bash
go run tests/rest_scripts/generateCommand.go \
  --awsAccessKeyId access \
  --awsSecretAccessKey secret \
  --client openssl \
  --commandType putObject \
  --bucketName test \
  --payload "hello" \
  --payloadType STREAMING-UNSIGNED-PAYLOAD-TRAILER \
  --chunkSize 8192 \
  --objectKey obj \
  --filePath req.txt \
  --checksumType crc64nvme
```

You can then send the request with:

```bash
openssl s_client -connect 127.0.0.1:7070 -ign_eof < req.txt > response.raw
```

tests/rest_scripts/command/payloadChunkedAWS.go

@@ -2,14 +2,16 @@ package command
 
 import (
 	"encoding/hex"
-	"github.com/versity/versitygw/tests/rest_scripts/logger"
 	"strings"
+
+	"github.com/versity/versitygw/tests/rest_scripts/logger"
 )
 
 type PayloadChunkedAWS struct {
 	*PayloadChunked
 	serviceString      string
 	currentDateTime    string
+	yyyymmdd           string
 	lastSignature      string
 	emptyByteSignature string
 	signingKey         []byte

tests/rest_scripts/command/payloadStreamingAWSHMACSHA256.go

@@ -5,22 +5,28 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"hash"
 	"io"
 	"os"
 )
 
+const (
+	streamPayloadTrailerAlgo = "AWS4-HMAC-SHA256-TRAILER"
+)
+
 type PayloadStreamingAWS4HMACSHA256 struct {
 	*PayloadChunkedAWS
+	hasher hash.Hash
 }
 
-func NewPayloadStreamingAWS4HMACSHA256(source DataSource, chunkSize int64, serviceString, currentDateTime string) *PayloadStreamingAWS4HMACSHA256 {
+func NewPayloadStreamingAWS4HMACSHA256(source DataSource, chunkSize int64, payloadType PayloadType, serviceString string, currentDateTime, yyyymmdd, checksumType string) *PayloadStreamingAWS4HMACSHA256 {
 	return &PayloadStreamingAWS4HMACSHA256{
 		PayloadChunkedAWS: &PayloadChunkedAWS{
 			PayloadChunked: &PayloadChunked{
 				Payload: &Payload{
 					dataSource:         source,
-					payloadType:        StreamingAWS4HMACSHA256Payload,
-					checksumType:       "",
+					payloadType:        payloadType,
+					checksumType:       checksumType,
 					dataSizeCalculated: false,
 					dataSize:           0,
 				},
@@ -28,6 +34,7 @@ func NewPayloadStreamingAWS4HMACSHA256(source DataSource, chunkSize int64, servi
 			},
 			serviceString:      serviceString,
 			currentDateTime:    currentDateTime,
+			yyyymmdd:           yyyymmdd,
 			lastSignature:      "",
 			emptyByteSignature: SHA256HashZeroBytes,
 			signingKey:         nil,
@@ -41,7 +48,19 @@ func (s *PayloadStreamingAWS4HMACSHA256) AddInitialSignatureAndSigningKey(initia
 }
 
 func (s *PayloadStreamingAWS4HMACSHA256) GetContentLength() (int64, error) {
-	return s.getChunkedPayloadContentLength(83, 85)
+	var trailerSize int64 = 85
+	if s.payloadType == StreamingAWS4HMACSHA256PayloadTrailer {
+		chLength, err := GetBase64ChecksumLength(s.checksumType)
+		if err != nil {
+			return 0, err
+		}
+
+		trailerSize += chLength + 92
+
+		// sum the checksum length
+		trailerSize += 16 + int64(len(s.checksumType))
+	}
+	return s.getChunkedPayloadContentLength(83, trailerSize)
 }
 
 func (s *PayloadStreamingAWS4HMACSHA256) addSignature(chunk []byte, outFile *os.File) error {
@@ -55,10 +74,70 @@ func (s *PayloadStreamingAWS4HMACSHA256) addSignature(chunk []byte, outFile *os.
 	return nil
 }
 
+func (s *PayloadStreamingAWS4HMACSHA256) addTrailer(outFile *os.File) error {
+	checksum, err := s.getBase64Checksum(s.hasher)
+	if err != nil {
+		return fmt.Errorf("failed to calculated the trailing checksum: %w", err)
+	}
+	tr := fmt.Sprintf("x-amz-checksum-%s", s.checksumType)
+	trailer := fmt.Sprintf("%s:%s", tr, checksum)
+
+	finalSig := s.calculateTrailerSignature(trailer)
+
+	trailerStr := fmt.Sprintf(
+		"\r\n%s\r\nx-amz-trailer-signature:%s",
+		trailer,
+		finalSig,
+	)
+
+	if _, err := outFile.Write([]byte(trailerStr)); err != nil {
+		return fmt.Errorf("error writing final chunk trailer: %w", err)
+	}
+
+	s.lastSignature = finalSig
+	return nil
+}
+
+func (s *PayloadStreamingAWS4HMACSHA256) calculateTrailerSignature(trailer string) string {
+	trailer += "\n"
+	strToSign := s.getTrailerChunkStringToSign(trailer)
+	return hex.EncodeToString(hmacSHA256(s.signingKey, strToSign))
+}
+
+func (s *PayloadStreamingAWS4HMACSHA256) getTrailerChunkStringToSign(trailer string) string {
+	hsh := sha256.Sum256([]byte(trailer))
+	sig := hex.EncodeToString(hsh[:])
+
+	prefix := s.getStringToSignPrefix(streamPayloadTrailerAlgo)
+
+	strToSign := fmt.Sprintf("%s\n%s\n%s",
+		prefix,
+		s.lastSignature,
+		sig,
+	)
+
+	return strToSign
+}
+
+func (s PayloadStreamingAWS4HMACSHA256) getStringToSignPrefix(algo string) string {
+	return fmt.Sprintf("%s\n%s\n%s",
+		algo,
+		s.currentDateTime,
+		s.serviceString,
+	)
+}
+
 func (s *PayloadStreamingAWS4HMACSHA256) getReader() (io.Reader, error) {
 	sourceFile, err := s.dataSource.GetReader()
 	if err != nil {
-		return nil, fmt.Errorf("error creating tee reader: %w", err)
+		return nil, fmt.Errorf("error creating reader: %w", err)
+	}
+	if s.payloadType == StreamingAWS4HMACSHA256PayloadTrailer && s.checksumType != "" {
+		s.hasher = s.getChecksumHasher()
+		sourceFile, err = s.dataSource.GetTeeReader(s.hasher)
+		if err != nil {
+			return nil, fmt.Errorf("error creating tee reader: %w", err)
+		}
 	}
 	return bufio.NewReader(sourceFile), nil
 }
@@ -66,8 +145,12 @@ func (s *PayloadStreamingAWS4HMACSHA256) getReader() (io.Reader, error) {
 func (s *PayloadStreamingAWS4HMACSHA256) WritePayload(filePath string) error {
 	s.addSignatureFunc = s.addSignature
 	s.getReaderFunc = s.getReader
-	s.addTrailerFunc = func(outFile *os.File) error {
-		return nil
+	if s.payloadType == StreamingAWS4HMACSHA256PayloadTrailer {
+		s.addTrailerFunc = s.addTrailer
+	} else {
+		s.addTrailerFunc = func(outFile *os.File) error {
+			return nil
+		}
 	}
 	return s.writeChunkedPayload(filePath)
 }

tests/rest_scripts/command/s3Command.go

@@ -7,11 +7,12 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
-	logger "github.com/versity/versitygw/tests/rest_scripts/logger"
 	"os"
 	"sort"
 	"strings"
 	"time"
+
+	logger "github.com/versity/versitygw/tests/rest_scripts/logger"
 )
 
 const (
@@ -128,10 +129,11 @@ func (s *S3Command) CurlShellCommand() (string, error) {
 }
 
 func (s *S3Command) prepareForBuild() error {
+	now := time.Now().UTC()
 	if s.IncorrectYearMonthDay {
-		s.currentDateTime = time.Now().Add(-48 * time.Hour).UTC().Format("20060102T150405Z")
+		s.currentDateTime = now.Add(-48 * time.Hour).Format("20060102T150405Z")
 	} else {
-		s.currentDateTime = time.Now().UTC().Format("20060102T150405Z")
+		s.currentDateTime = now.Format("20060102T150405Z")
 	}
 	protocolAndHost := strings.Split(s.Url, "://")
 	if len(protocolAndHost) != 2 {
@@ -184,9 +186,9 @@ func (s *S3Command) preparePayload() error {
 
 func (s *S3Command) initializeOpenSSLPayloadAndGetContentLength() error {
 	switch s.PayloadType {
-	case StreamingAWS4HMACSHA256Payload:
+	case StreamingAWS4HMACSHA256Payload, StreamingAWS4HMACSHA256PayloadTrailer:
 		serviceString := fmt.Sprintf("%s/%s/%s/aws4_request", s.yearMonthDay, s.AwsRegion, s.ServiceName)
-		s.payloadOpenSSL = NewPayloadStreamingAWS4HMACSHA256(s.dataSource, int64(s.ChunkSize), serviceString, s.currentDateTime)
+		s.payloadOpenSSL = NewPayloadStreamingAWS4HMACSHA256(s.dataSource, int64(s.ChunkSize), PayloadType(s.PayloadType), serviceString, s.currentDateTime, s.yearMonthDay, s.ChecksumType)
 	case StreamingUnsignedPayloadTrailer:
 		streamingUnsignedPayloadTrailerImpl := NewStreamingUnsignedPayloadWithTrailer(s.dataSource, int64(s.ChunkSize), s.ChecksumType)
 		streamingUnsignedPayloadTrailerImpl.OmitTrailerOrKey(s.OmitPayloadTrailer, s.OmitPayloadTrailerKey)
@@ -214,6 +216,9 @@ func (s *S3Command) addHeaderValues() error {
 	} else {
 		s.headerValues = append(s.headerValues, []string{"host", s.host})
 	}
+	if s.PayloadType == StreamingAWS4HMACSHA256PayloadTrailer && s.ChecksumType != "" {
+		s.headerValues = append(s.headerValues, []string{"x-amz-trailer", fmt.Sprintf("x-amz-checksum-%s", s.ChecksumType)})
+	}
 	s.headerValues = append(s.headerValues,
 		[]string{"x-amz-content-sha256", s.payloadHash},
 		[]string{"x-amz-date", s.currentDateTime},
@@ -418,7 +423,7 @@ func (s *S3Command) writeOpenSSLPayload(file *os.File) error {
 		awsPayload.AddInitialSignatureAndSigningKey(s.signature, s.signingKey)
 	}
 	switch s.PayloadType {
-	case UnsignedPayload, "", StreamingUnsignedPayloadTrailer, StreamingAWS4HMACSHA256Payload:
+	case UnsignedPayload, "", StreamingUnsignedPayloadTrailer, StreamingAWS4HMACSHA256Payload, StreamingAWS4HMACSHA256PayloadTrailer:
 		if err := s.payloadOpenSSL.WritePayload(s.FilePath); err != nil {
 			return fmt.Errorf("error writing payload to openssl file: %w", err)
 		}

tests/rest_scripts/generateCommand.go

@@ -4,10 +4,11 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/versity/versitygw/tests/rest_scripts/command"
-	logger "github.com/versity/versitygw/tests/rest_scripts/logger"
 	"log"
 	"strings"
+
+	"github.com/versity/versitygw/tests/rest_scripts/command"
+	logger "github.com/versity/versitygw/tests/rest_scripts/logger"
 )
 
 const (
@@ -271,5 +272,25 @@ func validateConfig() error {
 	if *client == command.CURL && *filePath != "" {
 		return fmt.Errorf("writing to file not currently supported for curl commands")
 	}
+	if !isValidChecksumType(checksumType) {
+		return fmt.Errorf("invalid checksum type: %s", *checksumType)
+	}
 	return nil
 }
+
+func isValidChecksumType(input *string) bool {
+	if input == nil {
+		return true
+	}
+
+	// make is case insensitive
+	chType := strings.ToLower(*input)
+	checksumType = &chType
+
+	switch chType {
+	case "", command.ChecksumCRC32, command.ChecksumCRC32C, command.ChecksumCRC64NVME, command.ChecksumSHA1, command.ChecksumSHA256:
+		return true
+	default:
+		return false
+	}
+}