fix: refactor chunk reader for clearer buffer processing

This ass a processBuffer() call that will recursively processes
the chunk encoding within the buffer.

backend/azure/azure.go

@@ -1201,6 +1201,7 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete
 		return ptNumber - nextPtNumber
 	})
 
+	last := len(blockList.UncommittedBlocks) - 1
 	for i, block := range blockList.UncommittedBlocks {
 		ptNumber, err := decodeBlockId(*block.Name)
 		if err != nil {
@@ -1213,6 +1214,11 @@ func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.Complete
 		if *input.MultipartUpload.Parts[i].PartNumber != int32(ptNumber) {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
+		// all parts except the last need to be greater, thena
+		// the minimum allowed size (5 Mib)
+		if i < last && *block.Size < backend.MinPartSize {
+			return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall)
+		}
 		blockIds = append(blockIds, *block.Name)
 	}
 

backend/common.go

@@ -34,6 +34,9 @@ const (
 	// this is the media type for directories in AWS and Nextcloud
 	DirContentType     = "application/x-directory"
 	DefaultContentType = "binary/octet-stream"
+
+	// this is the minimum allowed size for mp parts
+	MinPartSize = 5 * 1024 * 1024
 )
 
 func IsValidBucketName(name string) bool { return true }

backend/posix/posix.go

@@ -1375,7 +1375,6 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 
 	// check all parts ok
 	last := len(parts) - 1
-	partsize := int64(0)
 	var totalsize int64
 	for i, part := range parts {
 		if part.PartNumber == nil || *part.PartNumber < 1 {
@@ -1389,13 +1388,11 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
 		}
 
-		if i == 0 {
-			partsize = fi.Size()
-		}
 		totalsize += fi.Size()
-		// all parts except the last need to be the same size
-		if i < last && partsize != fi.Size() {
-			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
+		// all parts except the last need to be greater, thena
+		// the minimum allowed size (5 Mib)
+		if i < last && fi.Size() < backend.MinPartSize {
+			return nil, s3err.GetAPIError(s3err.ErrEntityTooSmall)
 		}
 
 		b, err := p.meta.RetrieveAttribute(nil, bucket, partObjPath, etagkey)

go.mod

@@ -7,8 +7,8 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
 	github.com/DataDog/datadog-go/v5 v5.6.0
-	github.com/aws/aws-sdk-go-v2 v1.34.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1
+	github.com/aws/aws-sdk-go-v2 v1.36.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2
 	github.com/aws/smithy-go v1.22.2
 	github.com/go-ldap/ldap/v3 v3.4.10
 	github.com/gofiber/fiber/v2 v2.52.6
@@ -30,13 +30,13 @@ require (
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -59,16 +59,16 @@ require (
 require (
 	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.29.2
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.55
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.29.4
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.57
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect

go.sum

@@ -14,8 +14,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3 h1:H5xDQaE3XowWfhZRUpnfC+rGZMEVoSiji+b+/HFAPU4=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw=
 github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
@@ -25,42 +25,42 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU=
-github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM=
+github.com/aws/aws-sdk-go-v2 v1.36.0 h1:b1wM5CcE65Ujwn565qcwgtOTT1aT4ADOHHgglKjG7fk=
+github.com/aws/aws-sdk-go-v2 v1.36.0/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg=
-github.com/aws/aws-sdk-go-v2/config v1.29.2 h1:JuIxOEPcSKpMB0J+khMjznG9LIhIBdmqNiEcPclnwqc=
-github.com/aws/aws-sdk-go-v2/config v1.29.2/go.mod h1:HktTHregOZwNSM/e7WTfVSu9RCX+3eOv+6ij27PtaYs=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.55 h1:CDhKnDEaGkLA5ZszV/qw5uwN5M8rbv9Cl0JRN+PRsaM=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.55/go.mod h1:kPD/vj+RB5MREDUky376+zdnjZpR+WgdBBvwrmnlmKE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 h1:kU7tmXNaJ07LsyN3BUgGqAmVmQtq0w6duVIHAKfp0/w=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25/go.mod h1:OiC8+OiqrURb1wrwmr/UbOVLFSWEGxjinj5C299VQdo=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54 h1:6BWOAho3Cgdy4cmNJ4HWY8VZgqODEU7Gw78XXireNZI=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54/go.mod h1:n+t/oyYErOV3jf/GxNTVlizSM9RMV1yH7jvcIvld3Do=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI=
+github.com/aws/aws-sdk-go-v2/config v1.29.4 h1:ObNqKsDYFGr2WxnoXKOhCvTlf3HhwtoGgc+KmZ4H5yg=
+github.com/aws/aws-sdk-go-v2/config v1.29.4/go.mod h1:j2/AF7j/qxVmsNIChw1tWfsVKOayJoGRDjg1Tgq7NPk=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.57 h1:kFQDsbdBAR3GZsB8xA+51ptEnq9TIj3tS4MuP5b+TcQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.57/go.mod h1:2kerxPUUbTagAr/kkaHiqvj/bcYHzi2qiJS/ZinllU0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 h1:7lOW8NUwE9UZekS1DYoiPdVAqZ6A+LheHWb+mHbNOq8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27/go.mod h1:w1BASFIPOPUae7AgaH4SbjNbfdkxuggLyGfNFTn8ITY=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57 h1:4hFrvTb32jty/LpKdIwWhMgqITPxNo9l1X1hjUyVCZ4=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57/go.mod h1:n6n8rfggAVPgDVldL1zk9QUzIWImRb6OWI8t9CfDImM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 h1:lWm9ucLSRFiI4dQQafLrEOmEDGry3Swrz0BIRdiHJqQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31/go.mod h1:Huu6GG0YTfbPphQkDSo4dEGmQRTKb9k9G7RdtyQWxuI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 h1:ACxDklUKKXb48+eg5ROZXi1vDgfMyfIA/WyvqHcHI0o=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31/go.mod h1:yadnfsDwqXeVaohbGc/RaD287PuyRw2wugkh5ZL2J6k=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 h1:8IwBjuLdqIO1dGB+dZ9zJEl8wzY3bVYxcs0Xyu/Lsc0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31/go.mod h1:8tMBcuVjL4kP/ECEIWTCWtwV2kj6+ouEKl4cqR4iWLw=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 h1:9LawY3cDJ3HE+v2GMd5SOkNLDwgN4K7TsCjyVBYu/L4=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 h1:kznaW4f81mNMlREkU9w3jUuJvU5g/KsqDV43ab7Rp6s=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.12/go.mod h1:bZy9r8e0/s0P7BSDHgMLXK2KvdyRRBIQ2blKlvLt0IU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 h1:mUwIpAvILeKFnRx4h1dEgGEFGuV8KJ3pEScZWVFYuZA=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11/go.mod h1:JDJtD+b8HNVv71axz8+S5492KM8wTzHRFpMKQbPlYxw=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 h1:g9d+TOsu3ac7SgmY2dUf1qMgu/uJVTlQ4VCbH6hRxSw=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.10/go.mod h1:WZfNmntu92HO44MVZAubQaz3qCuIdeOdog2sADfU6hU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 h1:siiQ+jummya9OLPDEyHVb2dLW4aOMe22FGDd0sAfuSw=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5/go.mod h1:iHVx2J9pWzITdP5MJY6qWfG34TfD9EA+Qi3eV6qQCXw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 h1:O+8vD2rGjfihBewr5bT+QUfYUHIxCVgG61LHoT59shM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12/go.mod h1:usVdWJaosa66NMvmCrr08NcWDBRv4E6+YFG2pUdw1Lk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 h1:tkVNm99nkJnFo1H9IIQb5QkCiPcvCDn3Pos+IeTbGRA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12/go.mod h1:dIVlquSPUMqEJtx2/W17SM2SuESRaVEhEV9alcMqxjw=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2 h1:dyC+iA2+Yc7iDMDh0R4eT6fi8TgBduc+BOWCy6Br0/o=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2/go.mod h1:FHSHmyEUkzRbaFFqqm6bkLAOQHgqhsLmfCahvCBMiyA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 h1:c5WJ3iHz7rLIgArznb3JCSQT3uUMiz9DLZhIX+1G8ok=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.14/go.mod h1:+JJQTxB6N4niArC14YNtxcQtwEqzS3o9Z32n7q33Rfs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 h1:f1L/JtUkVODD+k1+IiSJUUv8A++2qVr+Xvb3xWXETMU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13/go.mod h1:tvqlFoja8/s0o+UruA1Nrezo/df0PzdunMDDurUfg6U=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 h1:fqg6c1KVrc3SYWma/egWue5rKI4G2+M4wMQN2JosNAA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.12/go.mod h1:7Yn+p66q/jt38qMoVfNvjbm3D89mGBnkwDcijgtih8w=
 github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
 github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=

s3api/utils/signed-chunk-reader.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"hash"
 	"io"
-	"math"
 	"strconv"
 	"time"
 
@@ -79,22 +78,46 @@ func NewSignedChunkReader(r io.Reader, authdata AuthData, region, secret string,
 func (cr *ChunkReader) Read(p []byte) (int, error) {
 	n, err := cr.r.Read(p)
 	if err != nil && err != io.EOF {
-		return n, err
+		return 0, err
 	}
 
-	if cr.chunkDataLeft < int64(n) {
-		chunkSize := cr.chunkDataLeft
-		if chunkSize > 0 {
-			cr.chunkHash.Write(p[:chunkSize])
+	if n == 0 {
+		return 0, io.EOF
+	}
+
+	return cr.processBuffer(p[:n])
+}
+
+func (cr *ChunkReader) processBuffer(p []byte) (int, error) {
+	if len(p) == 0 {
+		return 0, nil
+	}
+
+	if cr.chunkDataLeft > 0 {
+		dataRemaining := cr.chunkDataLeft
+		if int64(len(p)) < cr.chunkDataLeft {
+			dataRemaining = int64(len(p))
 		}
-		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
-		n += int(chunkSize)
-		return n, err
+		cr.chunkDataLeft -= dataRemaining
+		cr.chunkHash.Write(p[:dataRemaining])
+		n, err := cr.processBuffer(p[dataRemaining:])
+		return int(dataRemaining) + n, err
 	}
 
-	cr.chunkDataLeft -= int64(n)
-	cr.chunkHash.Write(p[:n])
-	return n, err
+	shrunk, err := cr.parseAndRemoveChunkInfo(p)
+	if err != nil {
+		return 0, err
+	}
+	p = p[:len(p)-shrunk]
+	dataRemaining := cr.chunkDataLeft
+	if int64(len(p)) < cr.chunkDataLeft {
+		dataRemaining = int64(len(p))
+	}
+	cr.chunkDataLeft -= dataRemaining
+	cr.chunkHash.Write(p[:dataRemaining])
+
+	n, err := cr.processBuffer(p[dataRemaining:])
+	return int(dataRemaining) + n, err
 }
 
 // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
@@ -149,6 +172,8 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 		}
 	}
 
+	dataMoved := 0
+
 	if cr.trailerExpected != 0 {
 		if len(p) < len(chunkHdrDelim) {
 			// This is the special case where we need to consume the
@@ -162,22 +187,24 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 		// move data up to remove trailer
 		copy(p, p[cr.trailerExpected:])
 		n -= cr.trailerExpected
+		dataMoved += cr.trailerExpected
 	}
 
 	cr.skipcheck = false
 
 	chunkSize, sig, bufOffset, err := cr.parseChunkHeaderBytes(p[:n])
+	cr.chunkDataLeft = chunkSize
 	cr.currentChunkSize = chunkSize
 	cr.parsedSig = sig
 	if err == errskipHeader {
 		cr.chunkDataLeft = 0
-		return 0, nil
+		return dataMoved, nil
 	}
 	if err != nil {
-		return 0, err
+		return dataMoved, err
 	}
 	if chunkSize == 0 {
-		return 0, io.EOF
+		return dataMoved, io.EOF
 	}
 
 	cr.trailerExpected = len(chunkHdrDelim)
@@ -185,23 +212,9 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
 	// move data up to remove chunk header
 	copy(p, p[bufOffset:n])
 	n -= bufOffset
+	dataMoved += bufOffset
 
-	// if remaining buffer larger than chunk data,
-	// parse next header in buffer
-	if int64(n) > chunkSize {
-		cr.chunkDataLeft = 0
-		cr.chunkHash.Write(p[:chunkSize])
-		n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
-		if (chunkSize + int64(n)) > math.MaxInt {
-			return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
-		}
-		return n + int(chunkSize), err
-	}
-
-	cr.chunkDataLeft = chunkSize - int64(n)
-	cr.chunkHash.Write(p[:n])
-
-	return n, nil
+	return dataMoved, nil
 }
 
 // https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html

tests/integration/group-tests.go

@@ -321,6 +321,7 @@ func TestCompleteMultipartUpload(s *S3Conf) {
 	CompletedMultipartUpload_non_existing_bucket(s)
 	CompleteMultipartUpload_invalid_part_number(s)
 	CompleteMultipartUpload_invalid_ETag(s)
+	CompleteMultipartUpload_small_upload_size(s)
 	CompleteMultipartUpload_empty_parts(s)
 	CompleteMultipartUpload_success(s)
 	if !s.azureTests {
@@ -856,7 +857,6 @@ func GetIntTests() IntTests {
 		"CompletedMultipartUpload_non_existing_bucket":                        CompletedMultipartUpload_non_existing_bucket,
 		"CompleteMultipartUpload_invalid_part_number":                         CompleteMultipartUpload_invalid_part_number,
 		"CompleteMultipartUpload_invalid_ETag":                                CompleteMultipartUpload_invalid_ETag,
-		"CompleteMultipartUpload_empty_parts":                                 CompleteMultipartUpload_empty_parts,
 		"CompleteMultipartUpload_success":                                     CompleteMultipartUpload_success,
 		"CompleteMultipartUpload_racey_success":                               CompleteMultipartUpload_racey_success,
 		"PutBucketAcl_non_existing_bucket":                                    PutBucketAcl_non_existing_bucket,

tests/integration/tests.go

@@ -7192,6 +7192,49 @@ func CompleteMultipartUpload_invalid_ETag(s *S3Conf) error {
 	})
 }
 
+func CompleteMultipartUpload_small_upload_size(s *S3Conf) error {
+	testName := "CompleteMultipartUpload_small_upload_size"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		obj := "my-obj"
+
+		mp, err := createMp(s3client, bucket, obj)
+		if err != nil {
+			return err
+		}
+
+		// The uploaded parts size is 256 < 5 Mib (the minimum allowed size)
+		parts, _, err := uploadParts(s3client, 1024, 4, bucket, obj, *mp.UploadId)
+		if err != nil {
+			return err
+		}
+
+		cParts := []types.CompletedPart{}
+
+		for _, el := range parts {
+			cParts = append(cParts, types.CompletedPart{
+				PartNumber: el.PartNumber,
+				ETag:       el.ETag,
+			})
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+			Bucket:   &bucket,
+			Key:      &obj,
+			UploadId: mp.UploadId,
+			MultipartUpload: &types.CompletedMultipartUpload{
+				Parts: cParts,
+			},
+		})
+		cancel()
+		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrEntityTooSmall)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 func CompleteMultipartUpload_empty_parts(s *S3Conf) error {
 	testName := "CompleteMultipartUpload_empty_parts"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -7233,7 +7276,7 @@ func CompleteMultipartUpload_success(s *S3Conf) error {
 			return err
 		}
 
-		objSize := int64(5 * 1024 * 1024)
+		objSize := int64(25 * 1024 * 1024)
 		parts, csum, err := uploadParts(s3client, objSize, 5, bucket, obj, *out.UploadId)
 		if err != nil {
 			return err
@@ -7326,7 +7369,7 @@ func CompleteMultipartUpload_racey_success(s *S3Conf) error {
 		var mu sync.RWMutex
 		uploads := make([]mpinfo, 10)
 		sums := make([]string, 10)
-		objSize := int64(5 * 1024 * 1024)
+		objSize := int64(25 * 1024 * 1024)
 
 		eg := errgroup.Group{}
 		for i := 0; i < 10; i++ {
@@ -12990,7 +13033,7 @@ func Versioning_Multipart_Upload_success(s *S3Conf) error {
 			return err
 		}
 
-		objSize := int64(5 * 1024 * 1024)
+		objSize := int64(25 * 1024 * 1024)
 		parts, _, err := uploadParts(s3client, objSize, 5, bucket, obj, *out.UploadId)
 		if err != nil {
 			return err
@@ -13070,7 +13113,7 @@ func Versioning_Multipart_Upload_overwrite_an_object(s *S3Conf) error {
 			return err
 		}
 
-		objSize := int64(5 * 1024 * 1024)
+		objSize := int64(25 * 1024 * 1024)
 		parts, _, err := uploadParts(s3client, objSize, 5, bucket, obj, *out.UploadId)
 		if err != nil {
 			return err

tests/test_s3api_multipart.sh

@@ -54,7 +54,7 @@ source ./tests/commands/list_multipart_uploads.sh
   run create_test_files "$bucket_file"
   assert_success
 
-  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
+  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=20M count=1
   assert_success
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
@@ -115,7 +115,7 @@ source ./tests/commands/list_multipart_uploads.sh
   run create_test_file "$bucket_file"
   assert_success
 
-  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
+  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=20M count=1
   assert_success
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"

tests/util/util_multipart.sh

@@ -315,7 +315,7 @@ setup_multipart_upload_with_params() {
     return 1
   fi
 
-  if ! result=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER/$2" bs=5M count=1 2>&1); then
+  if ! result=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER/$2" bs=20M count=1 2>&1); then
     log 2 "error creating large file: $result"
     return 1
   fi