mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-19 22:44:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: more chunked upload tests with different payload types

Browse Source
This commit is contained in:
Luke McCrone
2025-11-13 11:25:32 -03:00
parent 8466d06371
commit b629f5d707
25 changed files with 1145 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/rest_scripts/command/s3Command.go
+186 -28
View File
@@ -15,7 +15,55 @@ import (
"time"
)
const (
CURL = "curl"
OPENSSL = "openssl"
)
const (
UnsignedPayload = "UNSIGNED-PAYLOAD"
StreamingAWS4HMACSHA256Payload = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
StreamingAWS4HMACSHA256PayloadTrailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
StreamingUnsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
StreamingAWS4ECDSAP256SHA256Payload = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
StreamingAWS4ECDSAP256SHA256PayloadTrailer = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
)
type PayloadType string
const (
ChecksumCRC32 = "crc32"
ChecksumCRC32C = "crc32c"
ChecksumCRC64NVME = "crc64nvme"
ChecksumSHA1 = "sha1"
ChecksumSHA256 = "sha256"
)
const SHA256HashZeroBytes = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
type S3RESTCommand struct {
Method string
Url string
Queries map[string]string
SignedParams map[string]string
UnsignedParams map[string]string
DataSource DataSource
}
type S3CommandErrors struct {
IncorrectSignature bool
AuthorizationHeaderMalformed bool
IncorrectCredential string
IncorrectYearMonthDay bool
InvalidYearMonthDay bool
IncorrectContentMD5 bool
MissingHostParam bool
CustomHostParam string
CustomHostParamSet bool
}
type S3Command struct {
Client string
Method string
Url string
BucketName string
@@ -36,11 +84,19 @@ type S3Command struct {
Payload string
ContentMD5 bool
IncorrectContentMD5 bool
CustomContentMD5 string
MissingHostParam bool
FilePath string
CustomHostParam string
CustomHostParamSet bool
PayloadType string
ChunkSize int
ChecksumType string
OmitPayloadTrailer bool
OmitPayloadTrailerKey bool
OmitContentLength bool
dataSource DataSource
currentDateTime string
host string
payloadHash string
@@ -50,6 +106,9 @@ type S3Command struct {
signedParamString string
yearMonthDay string
signature string
signingKey []byte
contentLength int64
payloadOpenSSL OpenSSLPayloadManager
}
func (s *S3Command) OpenSSLCommand() error {
@@ -73,9 +132,6 @@ func (s *S3Command) CurlShellCommand() (string, error) {
}
func (s *S3Command) prepareForBuild() error {
if s.PayloadFile != "" && s.Payload != "" {
return fmt.Errorf("cannot have both payload and payloadFile parameters set")
}
if s.IncorrectYearMonthDay {
s.currentDateTime = time.Now().Add(-48 * time.Hour).UTC().Format("20060102T150405Z")
} else {
@@ -86,24 +142,74 @@ func (s *S3Command) prepareForBuild() error {
return fmt.Errorf("invalid URL value: %s", s.Url)
}
s.host = protocolAndHost[1]
s.payloadHash = "UNSIGNED-PAYLOAD"
if err := s.addHeaderValues(); err != nil {
return fmt.Errorf("error adding header values: %w", err)
s.yearMonthDay = strings.Split(s.currentDateTime, "T")[0]
if s.InvalidYearMonthDay {
s.yearMonthDay = s.yearMonthDay[:len(s.yearMonthDay)-2]
}
s.path = "/" + s.BucketName
if s.ObjectKey != "" {
s.path += "/" + s.ObjectKey
}
s.generateCanonicalRequestString()
s.yearMonthDay = strings.Split(s.currentDateTime, "T")[0]
if s.InvalidYearMonthDay {
s.yearMonthDay = s.yearMonthDay[:len(s.yearMonthDay)-2]
if err := s.preparePayload(); err != nil {
return fmt.Errorf("error preparing payload: %w", err)
}
if err := s.addHeaderValues(); err != nil {
return fmt.Errorf("error adding header values: %w", err)
}
s.generateCanonicalRequestString()
s.getStsSignature()
return nil
}
func (s *S3Command) preparePayload() error {
if s.PayloadFile != "" && s.Payload != "" {
return fmt.Errorf("cannot have both payload and payloadFile parameters set")
}
if s.PayloadFile != "" {
s.dataSource = NewFileDataSource(s.PayloadFile)
} else if s.Payload != "" {
s.dataSource = NewStringDataSource(s.Payload)
}
if s.PayloadType != "" {
s.payloadHash = s.PayloadType
} else if s.dataSource != nil {
var err error
s.payloadHash, err = s.dataSource.CalculateSHA256HashString()
if err != nil {
return fmt.Errorf("error calculating sha256 hash")
}
} else {
s.payloadHash = SHA256HashZeroBytes
}
if s.Client == OPENSSL {
if err := s.initializeOpenSSLPayloadAndGetContentLength(); err != nil {
return fmt.Errorf("error initializing openssl payload: %w", err)
}
}
return nil
}
func (s *S3Command) initializeOpenSSLPayloadAndGetContentLength() error {
switch s.PayloadType {
case StreamingAWS4HMACSHA256Payload:
serviceString := fmt.Sprintf("%s/%s/%s/aws4_request", s.yearMonthDay, s.AwsRegion, s.ServiceName)
s.payloadOpenSSL = NewPayloadStreamingAWS4HMACSHA256(s.dataSource, int64(s.ChunkSize), serviceString, s.currentDateTime)
case StreamingUnsignedPayloadTrailer:
streamingUnsignedPayloadTrailerImpl := NewStreamingUnsignedPayloadWithTrailer(s.dataSource, int64(s.ChunkSize), s.ChecksumType)
streamingUnsignedPayloadTrailerImpl.OmitTrailerOrKey(s.OmitPayloadTrailer, s.OmitPayloadTrailerKey)
s.payloadOpenSSL = streamingUnsignedPayloadTrailerImpl
default:
return fmt.Errorf("unsupported OpenSSL payload type: '%s'", s.PayloadType)
}
var err error
s.contentLength, err = s.payloadOpenSSL.GetContentLength()
if err != nil {
return fmt.Errorf("error calculating Content-Length: %w", err)
}
logger.PrintDebug("Predicted payload size: %d", s.contentLength)
return nil
}
func (s *S3Command) addHeaderValues() error {
s.headerValues = [][]string{}
if s.MissingHostParam {
@@ -117,10 +223,22 @@ func (s *S3Command) addHeaderValues() error {
[]string{"x-amz-content-sha256", s.payloadHash},
[]string{"x-amz-date", s.currentDateTime},
)
if s.Client == OPENSSL && !s.OmitContentLength {
s.headerValues = append(s.headerValues,
[]string{"Content-Length", fmt.Sprintf("%d", s.contentLength)})
}
if s.dataSource != nil && s.PayloadType != UnsignedPayload {
payloadSize, err := s.dataSource.SourceDataByteSize()
if err != nil {
return fmt.Errorf("error getting payload size: %w", err)
}
s.headerValues = append(s.headerValues,
[]string{"x-amz-decoded-content-length", fmt.Sprintf("%d", payloadSize)})
}
for key, value := range s.SignedParams {
s.headerValues = append(s.headerValues, []string{key, value})
}
if s.ContentMD5 || s.IncorrectContentMD5 {
if s.ContentMD5 || s.IncorrectContentMD5 || s.CustomContentMD5 != "" {
if err := s.addContentMD5Header(); err != nil {
return fmt.Errorf("error adding Content-MD5 header: %w", err)
}
@@ -132,6 +250,14 @@ func (s *S3Command) addHeaderValues() error {
return nil
}
func (s *S3Command) modifyHash(md5Hash []byte) {
if md5Hash[0] == 'a' {
md5Hash[0] = 'A'
} else {
md5Hash[0] = 'a'
}
}
func (s *S3Command) addContentMD5Header() error {
var payloadData []byte
var err error
@@ -144,17 +270,18 @@ func (s *S3Command) addContentMD5Header() error {
payloadData = []byte(strings.Replace(s.Payload, "\\", "", -1))
}
hasher := md5.New()
hasher.Write(payloadData)
md5Hash := hasher.Sum(nil)
if s.IncorrectContentMD5 {
if md5Hash[0] == 'a' {
md5Hash[0] = 'A'
} else {
md5Hash[0] = 'a'
var contentMD5 string
if s.CustomContentMD5 != "" {
contentMD5 = s.CustomContentMD5
} else {
hasher := md5.New()
hasher.Write(payloadData)
md5Hash := hasher.Sum(nil)
if s.IncorrectContentMD5 {
s.modifyHash(md5Hash)
}
contentMD5 = base64.StdEncoding.EncodeToString(md5Hash)
}
contentMD5 := base64.StdEncoding.EncodeToString(md5Hash)
s.headerValues = append(s.headerValues, []string{"Content-MD5", contentMD5})
return nil
@@ -198,10 +325,10 @@ func (s *S3Command) getStsSignature() {
dateKey := hmacSHA256([]byte("AWS4"+s.AwsSecretAccessKey), s.yearMonthDay)
dateRegionKey := hmacSHA256(dateKey, s.AwsRegion)
dateRegionServiceKey := hmacSHA256(dateRegionKey, s.ServiceName)
signingKey := hmacSHA256(dateRegionServiceKey, "aws4_request")
s.signingKey = hmacSHA256(dateRegionServiceKey, "aws4_request")
// Generate signature
signatureBytes := hmacSHA256(signingKey, stsDataString)
signatureBytes := hmacSHA256(s.signingKey, stsDataString)
if s.IncorrectSignature {
if signatureBytes[0] == 'a' {
signatureBytes[0] = 'A'
@@ -237,9 +364,12 @@ func (s *S3Command) buildCurlShellCommand() (string, error) {
if s.PayloadFile != "" {
curlCommand = append(curlCommand, "-T", s.PayloadFile)
} else if s.Payload != "" {
s.Payload = strings.Replace(s.Payload, "\"", "\\\"", -1)
curlCommand = append(curlCommand, "-H", "\"Content-Type: application/xml\"", "-d", fmt.Sprintf("\"%s\"", s.Payload))
}
return strings.Join(curlCommand, " "), nil
curlStringCommand := strings.Join(curlCommand, " ")
logger.PrintDebug("curl command: %s", curlStringCommand)
return curlStringCommand, nil
}
func (s *S3Command) buildAuthorizationString() string {
@@ -254,6 +384,9 @@ func (s *S3Command) buildAuthorizationString() string {
}
func (s *S3Command) buildOpenSSLCommand() error {
if s.Query != "" {
s.path += "?" + s.Query
}
openSSLCommand := []string{fmt.Sprintf("%s %s HTTP/1.1", s.Method, s.path)}
openSSLCommand = append(openSSLCommand, s.buildAuthorizationString())
for _, headerValue := range s.headerValues {
@@ -262,16 +395,41 @@ func (s *S3Command) buildOpenSSLCommand() error {
}
openSSLCommand = append(openSSLCommand, fmt.Sprintf("%s:%s", headerValue[0], headerValue[1]))
}
openSSLCommand = append(openSSLCommand, "\r\n")
var file *os.File
var err error
if file, err = os.Create(s.FilePath); err != nil {
file, err := os.Create(s.FilePath)
if err != nil {
return fmt.Errorf("error opening file: %w", err)
}
defer func() {
file.Close()
}()
openSSLCommandBytes := []byte(strings.Join(openSSLCommand, "\r\n"))
if _, err = file.Write(openSSLCommandBytes); err != nil {
return fmt.Errorf("error writing to file: %w", err)
}
if s.PayloadFile != "" || s.Payload != "" {
if err = s.writeOpenSSLPayload(file); err != nil {
return fmt.Errorf("error writing openssl payload: %w", err)
}
}
return nil
}
func (s *S3Command) writeOpenSSLPayload(file *os.File) error {
if _, err := file.Write([]byte{'\r', '\n', '\r', '\n'}); err != nil {
return fmt.Errorf("error writing to file: %w", err)
}
if awsPayload, ok := s.payloadOpenSSL.(*PayloadStreamingAWS4HMACSHA256); ok {
awsPayload.AddInitialSignatureAndSigningKey(s.signature, s.signingKey)
}
switch s.PayloadType {
case UnsignedPayload, "", StreamingUnsignedPayloadTrailer, StreamingAWS4HMACSHA256Payload:
if err := s.payloadOpenSSL.WritePayload(s.FilePath); err != nil {
return fmt.Errorf("error writing payload to openssl file: %w", err)
}
default:
return fmt.Errorf("unsupported payload type: %s", s.PayloadType)
}
return nil
}