test: more chunked upload tests with different payload types

tests/rest_scripts/command/dataSource.go

@@ -0,0 +1,11 @@
+package command
+
+import "io"
+
+type DataSource interface {
+	SourceDataByteSize() (int64, error)
+	CalculateSHA256HashString() (string, error)
+	Close() error
+	GetReader() (io.Reader, error)
+	GetTeeReader(io.Writer) (io.Reader, error)
+}

tests/rest_scripts/command/fileDataSource.go

@@ -0,0 +1,82 @@
+package command
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+)
+
+type FileDataSource struct {
+	filePath string
+	File     *os.File
+}
+
+func NewFileDataSource(filePath string) *FileDataSource {
+	return &FileDataSource{
+		filePath: filePath,
+		File:     nil,
+	}
+}
+
+func (f *FileDataSource) SourceDataByteSize() (int64, error) {
+	fileInfo, err := os.Stat(f.filePath)
+	if err != nil {
+		return 0, fmt.Errorf("error getting file info: %w", err)
+	}
+	return fileInfo.Size(), nil
+}
+
+func (f *FileDataSource) CalculateSHA256HashString() (string, error) {
+	file, err := os.Open(f.filePath)
+	if err != nil {
+		return "", fmt.Errorf("error opening payload file '%s': %w", f.filePath, err)
+	}
+	defer file.Close()
+
+	hasher := sha256.New()
+	if _, err = io.Copy(hasher, file); err != nil {
+		return "", fmt.Errorf("error copying file data of '%s' to hasher: %w", f.filePath, err)
+	}
+
+	hash := hasher.Sum(nil)
+	return hex.EncodeToString(hash), nil
+}
+
+func (f *FileDataSource) Close() error {
+	if f.File != nil {
+		err := f.File.Close()
+		f.File = nil
+		return err
+	}
+	return nil
+}
+
+func (f *FileDataSource) openFile() error {
+	var err error
+	f.File, err = os.OpenFile(f.filePath, os.O_RDONLY, 0600)
+	if err != nil {
+		return fmt.Errorf("error opening file: %w", err)
+	}
+	return nil
+}
+
+func (f *FileDataSource) GetReader() (io.Reader, error) {
+	if f.File == nil {
+		if err := f.openFile(); err != nil {
+			return nil, err
+		}
+	}
+	return f.File, nil
+}
+
+func (f *FileDataSource) GetTeeReader(checksumWriter io.Writer) (io.Reader, error) {
+	if f.File == nil {
+		if err := f.openFile(); err != nil {
+			return nil, err
+		}
+	}
+	r := io.TeeReader(f.File, checksumWriter)
+	return r, nil
+}

tests/rest_scripts/command/openSSLPayloadManager.go

@@ -0,0 +1,6 @@
+package command
+
+type OpenSSLPayloadManager interface {
+	GetContentLength() (int64, error)
+	WritePayload(string) error
+}

tests/rest_scripts/command/payload.go

@@ -0,0 +1,91 @@
+package command
+
+import (
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"hash/crc32"
+	"reflect"
+
+	"github.com/minio/crc64nvme"
+)
+
+type Payload struct {
+	dataSource         DataSource
+	payloadType        PayloadType
+	checksumType       string
+	dataSizeCalculated bool
+	dataSize           int64
+}
+
+func GetBase64ChecksumLength(checksumType string) (int64, error) {
+	switch checksumType {
+	case ChecksumCRC32, ChecksumCRC32C:
+		return 8, nil
+	case ChecksumSHA256:
+		return 44, nil
+	case ChecksumSHA1:
+		return 28, nil
+	case ChecksumCRC64NVME:
+		return 12, nil
+	}
+	return 0, errors.New("unrecognized checksum type: " + checksumType)
+}
+
+func (p *Payload) GetDataSize() (int64, error) {
+	if !p.dataSizeCalculated {
+		dataSize, err := p.dataSource.SourceDataByteSize()
+		if err != nil {
+			return 0, fmt.Errorf("error getting payload data size: %w", err)
+		}
+		p.dataSize = dataSize
+		p.dataSizeCalculated = true
+	}
+	return p.dataSize, nil
+}
+
+func (p *Payload) getChecksumHasher() hash.Hash {
+	switch p.checksumType {
+	case ChecksumSHA256:
+		return sha256.New()
+	case ChecksumSHA1:
+		return sha1.New()
+	case ChecksumCRC32:
+		return crc32.NewIEEE()
+	case ChecksumCRC32C:
+		return crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	case ChecksumCRC64NVME:
+		return crc64nvme.New()
+	}
+	return nil
+}
+
+func (p *Payload) getBase64Checksum(hasher hash.Hash) (string, error) {
+	switch p.checksumType {
+	case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32:
+		return base64.StdEncoding.EncodeToString(hasher.Sum(nil)), nil
+	case ChecksumCRC32C:
+		var b [4]byte
+		hasher32, ok := hasher.(hash.Hash32)
+		if !ok {
+			return "", fmt.Errorf("'%v' not a Hash32 interface", reflect.TypeOf(hasher).String())
+		}
+		sum := hasher32.Sum32()
+		binary.BigEndian.PutUint32(b[:], sum)
+		return base64.StdEncoding.EncodeToString(b[:]), nil
+	case ChecksumCRC64NVME:
+		var b [8]byte
+		hasher64, ok := hasher.(hash.Hash64)
+		if !ok {
+			return "", fmt.Errorf("'%v' not a Hash64 interface", reflect.TypeOf(hasher).String())
+		}
+		sum := hasher64.Sum64()
+		binary.BigEndian.PutUint64(b[:], sum)
+		return base64.StdEncoding.EncodeToString(b[:]), nil
+	}
+	return "", fmt.Errorf("invalid checksum type specified: '%s'", p.checksumType)
+}

tests/rest_scripts/command/payloadChunked.go

@@ -0,0 +1,101 @@
+package command
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+type PayloadChunked struct {
+	*Payload
+	chunkSize        int64
+	getReaderFunc    func() (io.Reader, error)
+	addSignatureFunc func(chunk []byte, outFile *os.File) error
+	addTrailerFunc   func(outFile *os.File) error
+}
+
+func (c *PayloadChunked) getChunkedPayloadContentLength(additionalChunkHeaderSize, additionalTrailerSize int64) (int64, error) {
+	payloadSize, err := c.Payload.GetDataSize()
+	if err != nil {
+		return 0, fmt.Errorf("error getting payload data size: %w", err)
+	}
+	var sizeIdx int64
+	var contentLength int64
+	for sizeIdx = 0; sizeIdx < payloadSize; sizeIdx += c.chunkSize {
+		var endIdx int64
+		if sizeIdx+c.chunkSize < payloadSize {
+			endIdx = sizeIdx + c.chunkSize
+		} else {
+			endIdx = payloadSize
+		}
+		hexSize := fmt.Sprintf("%x", endIdx-sizeIdx)
+		contentLength += int64(len(hexSize)) + additionalChunkHeaderSize + (endIdx - sizeIdx) + 2
+	}
+	contentLength += 1 + additionalTrailerSize
+	return contentLength, nil
+}
+
+func (c *PayloadChunked) writeChunkedPayload(filePath string) error {
+	defer func() {
+		c.dataSource.Close()
+	}()
+	outFile, err := os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY, 0600)
+	if err != nil {
+		return fmt.Errorf("error writing to file: %w", err)
+	}
+	br, err := c.getReaderFunc()
+	if err != nil {
+		return fmt.Errorf("error getting data reader: %w", err)
+	}
+	payloadBuffer := make([]byte, c.chunkSize)
+	for {
+		var bytesRead int
+		if bytesRead, err = c.addChunk(br, payloadBuffer, outFile); err != nil {
+			return fmt.Errorf("error adding chunk: %w", err)
+		}
+		if bytesRead == 0 {
+			break
+		}
+	}
+	if _, err = outFile.Write([]byte{'0'}); err != nil {
+		return fmt.Errorf("error writing \\r\\n: %w", err)
+	}
+	if err = c.addSignatureFunc(nil, outFile); err != nil {
+		return fmt.Errorf("error adding signature: %w", err)
+	}
+	if err = c.addTrailerFunc(outFile); err != nil {
+		return fmt.Errorf("error adding trailer: %w", err)
+	}
+	if _, err = outFile.Write([]byte{'\r', '\n', '\r', '\n'}); err != nil {
+		return fmt.Errorf("error writing \\r\\n: %w", err)
+	}
+	return nil
+}
+
+func (c *PayloadChunked) addChunk(reader io.Reader, payloadBuffer []byte, outFile *os.File) (int, error) {
+	var bytesRead int
+	bytesRead, err := reader.Read(payloadBuffer)
+	if err != nil && err != io.EOF {
+		return 0, fmt.Errorf("error reading bytes: %w", err)
+	}
+	if bytesRead == 0 {
+		return 0, nil
+	}
+	hexString := fmt.Sprintf("%x", bytesRead)
+	if _, err = outFile.Write([]byte(hexString)); err != nil {
+		return 0, fmt.Errorf("error writing hex string: %w", err)
+	}
+	if err = c.addSignatureFunc(payloadBuffer[:bytesRead], outFile); err != nil {
+		return 0, fmt.Errorf("error adding signature: %w", err)
+	}
+	if _, err = outFile.Write([]byte{'\r', '\n'}); err != nil {
+		return 0, fmt.Errorf("error writing \\r\\n: %w", err)
+	}
+	if _, err = outFile.Write(payloadBuffer[:bytesRead]); err != nil {
+		return 0, fmt.Errorf("error writing bytes to file: %w", err)
+	}
+	if _, err = outFile.Write([]byte{'\r', '\n'}); err != nil {
+		return 0, fmt.Errorf("error writing \\r\\n: %w", err)
+	}
+	return bytesRead, nil
+}

tests/rest_scripts/command/payloadChunkedAWS.go

@@ -0,0 +1,28 @@
+package command
+
+import (
+	"encoding/hex"
+	"github.com/versity/versitygw/tests/rest_scripts/logger"
+	"strings"
+)
+
+type PayloadChunkedAWS struct {
+	*PayloadChunked
+	serviceString      string
+	currentDateTime    string
+	lastSignature      string
+	emptyByteSignature string
+	signingKey         []byte
+}
+
+func (c *PayloadChunkedAWS) getChunkedSTSSignature(chunkSignature string) string {
+	request := strings.Join([]string{"AWS4-HMAC-SHA256-PAYLOAD",
+		c.currentDateTime,
+		c.serviceString,
+		c.lastSignature,
+		c.emptyByteSignature,
+		chunkSignature}, "\n")
+	logger.PrintDebug("request: %s", request)
+	canonicalRequestHashBytes := hmacSHA256(c.signingKey, request)
+	return hex.EncodeToString(canonicalRequestHashBytes[:])
+}

tests/rest_scripts/command/payloadSizeCalculator.go

@@ -0,0 +1,5 @@
+package command
+
+type PayloadSizeCalculator interface {
+	CalculatePayloadSize() int64
+}

tests/rest_scripts/command/payloadStreamingAWSHMACSHA256.go

@@ -0,0 +1,73 @@
+package command
+
+import (
+	"bufio"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+)
+
+type PayloadStreamingAWS4HMACSHA256 struct {
+	*PayloadChunkedAWS
+}
+
+func NewPayloadStreamingAWS4HMACSHA256(source DataSource, chunkSize int64, serviceString, currentDateTime string) *PayloadStreamingAWS4HMACSHA256 {
+	return &PayloadStreamingAWS4HMACSHA256{
+		PayloadChunkedAWS: &PayloadChunkedAWS{
+			PayloadChunked: &PayloadChunked{
+				Payload: &Payload{
+					dataSource:         source,
+					payloadType:        StreamingAWS4HMACSHA256Payload,
+					checksumType:       "",
+					dataSizeCalculated: false,
+					dataSize:           0,
+				},
+				chunkSize: chunkSize,
+			},
+			serviceString:      serviceString,
+			currentDateTime:    currentDateTime,
+			lastSignature:      "",
+			emptyByteSignature: SHA256HashZeroBytes,
+			signingKey:         nil,
+		},
+	}
+}
+
+func (s *PayloadStreamingAWS4HMACSHA256) AddInitialSignatureAndSigningKey(initialSignature string, signingKey []byte) {
+	s.lastSignature = initialSignature
+	s.signingKey = signingKey
+}
+
+func (s *PayloadStreamingAWS4HMACSHA256) GetContentLength() (int64, error) {
+	return s.getChunkedPayloadContentLength(83, 85)
+}
+
+func (s *PayloadStreamingAWS4HMACSHA256) addSignature(chunk []byte, outFile *os.File) error {
+	sha256sum := sha256.Sum256(chunk)
+	sha256sumString := hex.EncodeToString(sha256sum[:])
+	signature := s.getChunkedSTSSignature(sha256sumString)
+	if _, err := outFile.Write([]byte(";chunk-signature=" + signature)); err != nil {
+		return fmt.Errorf("error writing chunked signature: %w", err)
+	}
+	s.lastSignature = signature
+	return nil
+}
+
+func (s *PayloadStreamingAWS4HMACSHA256) getReader() (io.Reader, error) {
+	sourceFile, err := s.dataSource.GetReader()
+	if err != nil {
+		return nil, fmt.Errorf("error creating tee reader: %w", err)
+	}
+	return bufio.NewReader(sourceFile), nil
+}
+
+func (s *PayloadStreamingAWS4HMACSHA256) WritePayload(filePath string) error {
+	s.addSignatureFunc = s.addSignature
+	s.getReaderFunc = s.getReader
+	s.addTrailerFunc = func(outFile *os.File) error {
+		return nil
+	}
+	return s.writeChunkedPayload(filePath)
+}

tests/rest_scripts/command/putBucketTaggingCommand.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
-	"strings"
 )
 
 type Tag struct {
@@ -51,7 +50,7 @@ func NewPutBucketTaggingCommand(s3Command *S3Command, fields *PutBucketTaggingFi
 		return nil, errors.New("tagCount can not be set simultaneously with tagKeys or tagValues")
 	}
 	command.Tags = &PutBucketTaggingTags{
-		XMLNamespace: "https://s3.amazonaws.com/doc/2006-03-01/",
+		XMLNamespace: "http://s3.amazonaws.com/doc/2006-03-01/",
 	}
 	if fields.TagCount > 0 {
 		command.Tags.GenerateKeyValuePairs(fields.TagCount)
@@ -65,8 +64,7 @@ func NewPutBucketTaggingCommand(s3Command *S3Command, fields *PutBucketTaggingFi
 	if err != nil {
 		return nil, fmt.Errorf("error marshalling XML: %w", err)
 	}
-	command.Payload = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + string(xmlData)
-	command.Payload = strings.Replace(command.Payload, "\"", "\\\"", -1)
+	command.Payload = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + string(xmlData)
 	return command, nil
 }
 

tests/rest_scripts/command/putObjectCommand.go

@@ -0,0 +1,17 @@
+package command
+
+import (
+	"errors"
+)
+
+func NewPutObjectCommand(s3Command *S3Command) (*S3Command, error) {
+	if s3Command.BucketName == "" {
+		return nil, errors.New("PutObject must have bucket name")
+	}
+	if s3Command.ObjectKey == "" {
+		return nil, errors.New("PutObject must have object key")
+	}
+	s3Command.Method = "PUT"
+	s3Command.Query = ""
+	return s3Command, nil
+}

tests/rest_scripts/command/s3Command.go

@@ -15,7 +15,55 @@ import (
 	"time"
 )
 
+const (
+	CURL    = "curl"
+	OPENSSL = "openssl"
+)
+
+const (
+	UnsignedPayload                            = "UNSIGNED-PAYLOAD"
+	StreamingAWS4HMACSHA256Payload             = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+	StreamingAWS4HMACSHA256PayloadTrailer      = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
+	StreamingUnsignedPayloadTrailer            = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
+	StreamingAWS4ECDSAP256SHA256Payload        = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
+	StreamingAWS4ECDSAP256SHA256PayloadTrailer = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
+)
+
+type PayloadType string
+
+const (
+	ChecksumCRC32     = "crc32"
+	ChecksumCRC32C    = "crc32c"
+	ChecksumCRC64NVME = "crc64nvme"
+	ChecksumSHA1      = "sha1"
+	ChecksumSHA256    = "sha256"
+)
+
+const SHA256HashZeroBytes = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+type S3RESTCommand struct {
+	Method         string
+	Url            string
+	Queries        map[string]string
+	SignedParams   map[string]string
+	UnsignedParams map[string]string
+	DataSource     DataSource
+}
+
+type S3CommandErrors struct {
+	IncorrectSignature           bool
+	AuthorizationHeaderMalformed bool
+	IncorrectCredential          string
+	IncorrectYearMonthDay        bool
+	InvalidYearMonthDay          bool
+	IncorrectContentMD5          bool
+	MissingHostParam             bool
+	CustomHostParam              string
+	CustomHostParamSet           bool
+}
+
 type S3Command struct {
+	Client                       string
 	Method                       string
 	Url                          string
 	BucketName                   string
@@ -36,11 +84,19 @@ type S3Command struct {
 	Payload                      string
 	ContentMD5                   bool
 	IncorrectContentMD5          bool
+	CustomContentMD5             string
 	MissingHostParam             bool
 	FilePath                     string
 	CustomHostParam              string
 	CustomHostParamSet           bool
+	PayloadType                  string
+	ChunkSize                    int
+	ChecksumType                 string
+	OmitPayloadTrailer           bool
+	OmitPayloadTrailerKey        bool
+	OmitContentLength            bool
 
+	dataSource           DataSource
 	currentDateTime      string
 	host                 string
 	payloadHash          string
@@ -50,6 +106,9 @@ type S3Command struct {
 	signedParamString    string
 	yearMonthDay         string
 	signature            string
+	signingKey           []byte
+	contentLength        int64
+	payloadOpenSSL       OpenSSLPayloadManager
 }
 
 func (s *S3Command) OpenSSLCommand() error {
@@ -73,9 +132,6 @@ func (s *S3Command) CurlShellCommand() (string, error) {
 }
 
 func (s *S3Command) prepareForBuild() error {
-	if s.PayloadFile != "" && s.Payload != "" {
-		return fmt.Errorf("cannot have both payload and payloadFile parameters set")
-	}
 	if s.IncorrectYearMonthDay {
 		s.currentDateTime = time.Now().Add(-48 * time.Hour).UTC().Format("20060102T150405Z")
 	} else {
@@ -86,24 +142,74 @@ func (s *S3Command) prepareForBuild() error {
 		return fmt.Errorf("invalid URL value: %s", s.Url)
 	}
 	s.host = protocolAndHost[1]
-	s.payloadHash = "UNSIGNED-PAYLOAD"
-	if err := s.addHeaderValues(); err != nil {
-		return fmt.Errorf("error adding header values: %w", err)
+	s.yearMonthDay = strings.Split(s.currentDateTime, "T")[0]
+	if s.InvalidYearMonthDay {
+		s.yearMonthDay = s.yearMonthDay[:len(s.yearMonthDay)-2]
 	}
 	s.path = "/" + s.BucketName
 	if s.ObjectKey != "" {
 		s.path += "/" + s.ObjectKey
 	}
-	s.generateCanonicalRequestString()
-
-	s.yearMonthDay = strings.Split(s.currentDateTime, "T")[0]
-	if s.InvalidYearMonthDay {
-		s.yearMonthDay = s.yearMonthDay[:len(s.yearMonthDay)-2]
+	if err := s.preparePayload(); err != nil {
+		return fmt.Errorf("error preparing payload: %w", err)
 	}
+	if err := s.addHeaderValues(); err != nil {
+		return fmt.Errorf("error adding header values: %w", err)
+	}
+	s.generateCanonicalRequestString()
 	s.getStsSignature()
 	return nil
 }
 
+func (s *S3Command) preparePayload() error {
+	if s.PayloadFile != "" && s.Payload != "" {
+		return fmt.Errorf("cannot have both payload and payloadFile parameters set")
+	}
+	if s.PayloadFile != "" {
+		s.dataSource = NewFileDataSource(s.PayloadFile)
+	} else if s.Payload != "" {
+		s.dataSource = NewStringDataSource(s.Payload)
+	}
+	if s.PayloadType != "" {
+		s.payloadHash = s.PayloadType
+	} else if s.dataSource != nil {
+		var err error
+		s.payloadHash, err = s.dataSource.CalculateSHA256HashString()
+		if err != nil {
+			return fmt.Errorf("error calculating sha256 hash")
+		}
+	} else {
+		s.payloadHash = SHA256HashZeroBytes
+	}
+	if s.Client == OPENSSL {
+		if err := s.initializeOpenSSLPayloadAndGetContentLength(); err != nil {
+			return fmt.Errorf("error initializing openssl payload: %w", err)
+		}
+	}
+	return nil
+}
+
+func (s *S3Command) initializeOpenSSLPayloadAndGetContentLength() error {
+	switch s.PayloadType {
+	case StreamingAWS4HMACSHA256Payload:
+		serviceString := fmt.Sprintf("%s/%s/%s/aws4_request", s.yearMonthDay, s.AwsRegion, s.ServiceName)
+		s.payloadOpenSSL = NewPayloadStreamingAWS4HMACSHA256(s.dataSource, int64(s.ChunkSize), serviceString, s.currentDateTime)
+	case StreamingUnsignedPayloadTrailer:
+		streamingUnsignedPayloadTrailerImpl := NewStreamingUnsignedPayloadWithTrailer(s.dataSource, int64(s.ChunkSize), s.ChecksumType)
+		streamingUnsignedPayloadTrailerImpl.OmitTrailerOrKey(s.OmitPayloadTrailer, s.OmitPayloadTrailerKey)
+		s.payloadOpenSSL = streamingUnsignedPayloadTrailerImpl
+	default:
+		return fmt.Errorf("unsupported OpenSSL payload type: '%s'", s.PayloadType)
+	}
+	var err error
+	s.contentLength, err = s.payloadOpenSSL.GetContentLength()
+	if err != nil {
+		return fmt.Errorf("error calculating Content-Length: %w", err)
+	}
+	logger.PrintDebug("Predicted payload size: %d", s.contentLength)
+	return nil
+}
+
 func (s *S3Command) addHeaderValues() error {
 	s.headerValues = [][]string{}
 	if s.MissingHostParam {
@@ -117,10 +223,22 @@ func (s *S3Command) addHeaderValues() error {
 		[]string{"x-amz-content-sha256", s.payloadHash},
 		[]string{"x-amz-date", s.currentDateTime},
 	)
+	if s.Client == OPENSSL && !s.OmitContentLength {
+		s.headerValues = append(s.headerValues,
+			[]string{"Content-Length", fmt.Sprintf("%d", s.contentLength)})
+	}
+	if s.dataSource != nil && s.PayloadType != UnsignedPayload {
+		payloadSize, err := s.dataSource.SourceDataByteSize()
+		if err != nil {
+			return fmt.Errorf("error getting payload size: %w", err)
+		}
+		s.headerValues = append(s.headerValues,
+			[]string{"x-amz-decoded-content-length", fmt.Sprintf("%d", payloadSize)})
+	}
 	for key, value := range s.SignedParams {
 		s.headerValues = append(s.headerValues, []string{key, value})
 	}
-	if s.ContentMD5 || s.IncorrectContentMD5 {
+	if s.ContentMD5 || s.IncorrectContentMD5 || s.CustomContentMD5 != "" {
 		if err := s.addContentMD5Header(); err != nil {
 			return fmt.Errorf("error adding Content-MD5 header: %w", err)
 		}
@@ -132,6 +250,14 @@ func (s *S3Command) addHeaderValues() error {
 	return nil
 }
 
+func (s *S3Command) modifyHash(md5Hash []byte) {
+	if md5Hash[0] == 'a' {
+		md5Hash[0] = 'A'
+	} else {
+		md5Hash[0] = 'a'
+	}
+}
+
 func (s *S3Command) addContentMD5Header() error {
 	var payloadData []byte
 	var err error
@@ -144,17 +270,18 @@ func (s *S3Command) addContentMD5Header() error {
 		payloadData = []byte(strings.Replace(s.Payload, "\\", "", -1))
 	}
 
-	hasher := md5.New()
-	hasher.Write(payloadData)
-	md5Hash := hasher.Sum(nil)
-	if s.IncorrectContentMD5 {
-		if md5Hash[0] == 'a' {
-			md5Hash[0] = 'A'
-		} else {
-			md5Hash[0] = 'a'
+	var contentMD5 string
+	if s.CustomContentMD5 != "" {
+		contentMD5 = s.CustomContentMD5
+	} else {
+		hasher := md5.New()
+		hasher.Write(payloadData)
+		md5Hash := hasher.Sum(nil)
+		if s.IncorrectContentMD5 {
+			s.modifyHash(md5Hash)
 		}
+		contentMD5 = base64.StdEncoding.EncodeToString(md5Hash)
 	}
-	contentMD5 := base64.StdEncoding.EncodeToString(md5Hash)
 
 	s.headerValues = append(s.headerValues, []string{"Content-MD5", contentMD5})
 	return nil
@@ -198,10 +325,10 @@ func (s *S3Command) getStsSignature() {
 	dateKey := hmacSHA256([]byte("AWS4"+s.AwsSecretAccessKey), s.yearMonthDay)
 	dateRegionKey := hmacSHA256(dateKey, s.AwsRegion)
 	dateRegionServiceKey := hmacSHA256(dateRegionKey, s.ServiceName)
-	signingKey := hmacSHA256(dateRegionServiceKey, "aws4_request")
+	s.signingKey = hmacSHA256(dateRegionServiceKey, "aws4_request")
 
 	// Generate signature
-	signatureBytes := hmacSHA256(signingKey, stsDataString)
+	signatureBytes := hmacSHA256(s.signingKey, stsDataString)
 	if s.IncorrectSignature {
 		if signatureBytes[0] == 'a' {
 			signatureBytes[0] = 'A'
@@ -237,9 +364,12 @@ func (s *S3Command) buildCurlShellCommand() (string, error) {
 	if s.PayloadFile != "" {
 		curlCommand = append(curlCommand, "-T", s.PayloadFile)
 	} else if s.Payload != "" {
+		s.Payload = strings.Replace(s.Payload, "\"", "\\\"", -1)
 		curlCommand = append(curlCommand, "-H", "\"Content-Type: application/xml\"", "-d", fmt.Sprintf("\"%s\"", s.Payload))
 	}
-	return strings.Join(curlCommand, " "), nil
+	curlStringCommand := strings.Join(curlCommand, " ")
+	logger.PrintDebug("curl command: %s", curlStringCommand)
+	return curlStringCommand, nil
 }
 
 func (s *S3Command) buildAuthorizationString() string {
@@ -254,6 +384,9 @@ func (s *S3Command) buildAuthorizationString() string {
 }
 
 func (s *S3Command) buildOpenSSLCommand() error {
+	if s.Query != "" {
+		s.path += "?" + s.Query
+	}
 	openSSLCommand := []string{fmt.Sprintf("%s %s HTTP/1.1", s.Method, s.path)}
 	openSSLCommand = append(openSSLCommand, s.buildAuthorizationString())
 	for _, headerValue := range s.headerValues {
@@ -262,16 +395,41 @@ func (s *S3Command) buildOpenSSLCommand() error {
 		}
 		openSSLCommand = append(openSSLCommand, fmt.Sprintf("%s:%s", headerValue[0], headerValue[1]))
 	}
-	openSSLCommand = append(openSSLCommand, "\r\n")
-	var file *os.File
-	var err error
-	if file, err = os.Create(s.FilePath); err != nil {
+
+	file, err := os.Create(s.FilePath)
+	if err != nil {
 		return fmt.Errorf("error opening file: %w", err)
 	}
+	defer func() {
+		file.Close()
+	}()
 	openSSLCommandBytes := []byte(strings.Join(openSSLCommand, "\r\n"))
 	if _, err = file.Write(openSSLCommandBytes); err != nil {
 		return fmt.Errorf("error writing to file: %w", err)
 	}
+	if s.PayloadFile != "" || s.Payload != "" {
+		if err = s.writeOpenSSLPayload(file); err != nil {
+			return fmt.Errorf("error writing openssl payload: %w", err)
+		}
+	}
+	return nil
+}
+
+func (s *S3Command) writeOpenSSLPayload(file *os.File) error {
+	if _, err := file.Write([]byte{'\r', '\n', '\r', '\n'}); err != nil {
+		return fmt.Errorf("error writing to file: %w", err)
+	}
+	if awsPayload, ok := s.payloadOpenSSL.(*PayloadStreamingAWS4HMACSHA256); ok {
+		awsPayload.AddInitialSignatureAndSigningKey(s.signature, s.signingKey)
+	}
+	switch s.PayloadType {
+	case UnsignedPayload, "", StreamingUnsignedPayloadTrailer, StreamingAWS4HMACSHA256Payload:
+		if err := s.payloadOpenSSL.WritePayload(s.FilePath); err != nil {
+			return fmt.Errorf("error writing payload to openssl file: %w", err)
+		}
+	default:
+		return fmt.Errorf("unsupported payload type: %s", s.PayloadType)
+	}
 	return nil
 }
 

tests/rest_scripts/command/streamingUnsignedPayloadTrailer.go

@@ -0,0 +1,99 @@
+package command
+
+import (
+	"bufio"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+)
+
+type StreamingUnsignedPayloadWithTrailer struct {
+	*PayloadChunked
+	hasher         hash.Hash
+	checksumHeader string
+	checksumValue  string
+	omitTrailer    bool
+	omitTrailerKey bool
+}
+
+func NewStreamingUnsignedPayloadWithTrailer(source DataSource, chunkSize int64, checksumType string) *StreamingUnsignedPayloadWithTrailer {
+	return &StreamingUnsignedPayloadWithTrailer{
+		PayloadChunked: &PayloadChunked{
+			Payload: &Payload{
+				dataSource:         source,
+				payloadType:        StreamingUnsignedPayloadTrailer,
+				checksumType:       checksumType,
+				dataSizeCalculated: false,
+				dataSize:           0,
+			},
+			chunkSize: chunkSize,
+		},
+		checksumHeader: "x-amz-checksum-" + checksumType,
+		checksumValue:  "",
+		omitTrailer:    false,
+		omitTrailerKey: false,
+	}
+}
+
+func (s *StreamingUnsignedPayloadWithTrailer) OmitTrailerOrKey(omitTrailer, omitTrailerKey bool) {
+	s.omitTrailer = omitTrailer
+	s.omitTrailerKey = omitTrailerKey
+}
+
+func (s *StreamingUnsignedPayloadWithTrailer) GetContentLength() (int64, error) {
+	checksumValueLength, err := GetBase64ChecksumLength(s.checksumType)
+	if err != nil {
+		return 0, fmt.Errorf("error getting base64 checksum length: %w", err)
+	}
+	var trailerLength int64
+	if s.omitTrailer {
+		trailerLength = 4
+	} else if s.omitTrailerKey {
+		trailerLength = 1 + checksumValueLength + 4
+	} else {
+		trailerLength = 2 + int64(len(s.checksumHeader)) + 1 + checksumValueLength + 4
+	}
+	return s.getChunkedPayloadContentLength(2, trailerLength)
+}
+
+func (s *StreamingUnsignedPayloadWithTrailer) getReader() (io.Reader, error) {
+	s.hasher = s.getChecksumHasher()
+	teeReader, err := s.dataSource.GetTeeReader(s.hasher)
+	if err != nil {
+		return nil, fmt.Errorf("error creating tee reader: %w", err)
+	}
+	br := bufio.NewReader(teeReader)
+	return br, nil
+}
+
+func (s *StreamingUnsignedPayloadWithTrailer) addTrailer(outFile *os.File) error {
+	if s.omitTrailer {
+		return nil
+	}
+	if _, err := outFile.Write([]byte{'\r', '\n'}); err != nil {
+		return fmt.Errorf("error writing \\r\\n: %w", err)
+	}
+	checksum, err := s.getBase64Checksum(s.hasher)
+	if err != nil {
+		return fmt.Errorf("error getting checksum: %w", err)
+	}
+	if !s.omitTrailerKey {
+		if _, err = outFile.Write([]byte(s.checksumHeader)); err != nil {
+			return fmt.Errorf("error writing trailer key: %w", err)
+		}
+	}
+	if _, err = outFile.Write([]byte(":" + checksum)); err != nil {
+		return fmt.Errorf("error writing checksum: %w", err)
+	}
+	return nil
+}
+
+func (s *StreamingUnsignedPayloadWithTrailer) WritePayload(filePath string) error {
+	s.addSignatureFunc = func(chunk []byte, file *os.File) error {
+		return nil
+	}
+	s.getReaderFunc = s.getReader
+	s.addTrailerFunc = s.addTrailer
+	return s.writeChunkedPayload(filePath)
+}

tests/rest_scripts/command/stringDataSource.go

@@ -0,0 +1,42 @@
+package command
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"strings"
+)
+
+type StringDataSource struct {
+	dataString string
+}
+
+func NewStringDataSource(dataString string) *StringDataSource {
+	return &StringDataSource{
+		dataString: dataString,
+	}
+}
+
+func (s *StringDataSource) SourceDataByteSize() (int64, error) {
+	return int64(len(s.dataString)), nil
+}
+
+func (s *StringDataSource) CalculateSHA256HashString() (string, error) {
+	hash := sha256.Sum256([]byte(s.dataString))
+	return hex.EncodeToString(hash[:]), nil
+}
+
+func (s *StringDataSource) Close() error {
+	return nil
+}
+
+func (s *StringDataSource) GetReader() (io.Reader, error) {
+	stringReader := strings.NewReader(s.dataString)
+	return stringReader, nil
+}
+
+func (s *StringDataSource) GetTeeReader(checksumWriter io.Writer) (io.Reader, error) {
+	stringReader := strings.NewReader(s.dataString)
+	r := io.TeeReader(stringReader, checksumWriter)
+	return r, nil
+}

tests/rest_scripts/command/wholePayload.go

@@ -0,0 +1,56 @@
+package command
+
+import (
+	"fmt"
+	"os"
+)
+
+type WholePayload struct {
+	*Payload
+}
+
+func NewWholePayload(dataSource DataSource) *WholePayload {
+	return &WholePayload{
+		&Payload{
+			dataSource:         dataSource,
+			payloadType:        "",
+			checksumType:       "",
+			dataSizeCalculated: false,
+			dataSize:           0,
+		},
+	}
+}
+
+func (w *WholePayload) CalculatePayloadSize() (int64, error) {
+	return w.GetDataSize()
+}
+
+func (w *WholePayload) GetContentLength() (int64, error) {
+	return w.GetDataSize()
+}
+
+func (w *WholePayload) WritePayload(filePath string) error {
+	sourceFile, err := w.dataSource.GetReader()
+	if err != nil {
+		return fmt.Errorf("error creating tee reader: %w", err)
+	}
+	outFile, err := os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY, 0600)
+	if err != nil {
+		return fmt.Errorf("error writing to file: %w", err)
+	}
+	buffer := make([]byte, 256)
+	for {
+		var bytesRead int
+		bytesRead, err = sourceFile.Read(buffer)
+		if err != nil {
+			return fmt.Errorf("error reading data bytes: %w", err)
+		}
+		if bytesRead == 0 {
+			break
+		}
+		if _, err = outFile.Write(buffer[:bytesRead]); err != nil {
+			return fmt.Errorf("error writing bytes to file: %w", err)
+		}
+	}
+	return nil
+}