mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-07-19 06:22:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums

Browse Source 
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
niksis02
2026-04-30 22:24:07 +04:00
committed by Ben McClelland
parent fbe2a4ba10
commit d2fa265fb8
39 changed files with 1811 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/posix/posix.go
+223 -56
View File
@@ -1659,11 +1659,79 @@ func getPartChecksum(algo types.ChecksumAlgorithm, part types.CompletedPart) str
return backend.GetStringFromPtr(part.ChecksumSHA256)
case types.ChecksumAlgorithmCrc64nvme:
return backend.GetStringFromPtr(part.ChecksumCRC64NVME)
case types.ChecksumAlgorithmSha512:
return backend.GetStringFromPtr(part.ChecksumSHA512)
case types.ChecksumAlgorithmMd5:
return backend.GetStringFromPtr(part.ChecksumMD5)
case types.ChecksumAlgorithmXxhash64:
return backend.GetStringFromPtr(part.ChecksumXXHASH64)
case types.ChecksumAlgorithmXxhash3:
return backend.GetStringFromPtr(part.ChecksumXXHASH3)
case types.ChecksumAlgorithmXxhash128:
return backend.GetStringFromPtr(part.ChecksumXXHASH128)
default:
return ""
}
}
func setStoredChecksum(checksum *s3response.Checksum, algo types.ChecksumAlgorithm, sum *string) {
if sum == nil {
return
}
switch algo {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = sum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = sum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = sum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = sum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = sum
case types.ChecksumAlgorithmSha512:
checksum.SHA512 = sum
case types.ChecksumAlgorithmMd5:
checksum.MD5 = sum
case types.ChecksumAlgorithmXxhash64:
checksum.XXHASH64 = sum
case types.ChecksumAlgorithmXxhash3:
checksum.XXHASH3 = sum
case types.ChecksumAlgorithmXxhash128:
checksum.XXHASH128 = sum
}
}
func setUploadPartChecksum(res *s3.UploadPartOutput, algo types.ChecksumAlgorithm, sum *string) {
if sum == nil {
return
}
switch algo {
case types.ChecksumAlgorithmCrc32:
res.ChecksumCRC32 = sum
case types.ChecksumAlgorithmCrc32c:
res.ChecksumCRC32C = sum
case types.ChecksumAlgorithmSha1:
res.ChecksumSHA1 = sum
case types.ChecksumAlgorithmSha256:
res.ChecksumSHA256 = sum
case types.ChecksumAlgorithmCrc64nvme:
res.ChecksumCRC64NVME = sum
case types.ChecksumAlgorithmSha512:
res.ChecksumSHA512 = sum
case types.ChecksumAlgorithmMd5:
res.ChecksumMD5 = sum
case types.ChecksumAlgorithmXxhash64:
res.ChecksumXXHASH64 = sum
case types.ChecksumAlgorithmXxhash3:
res.ChecksumXXHASH3 = sum
case types.ChecksumAlgorithmXxhash128:
res.ChecksumXXHASH128 = sum
}
}
func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
release, err := p.acquireActionSlot(ctx)
if err != nil {
@@ -1947,6 +2015,11 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
var sha1 *string
var sha256 *string
var crc64nvme *string
var sha512 *string
var md5sum *string
var xxhash64 *string
var xxhash3 *string
var xxhash128 *string
var gotSum *string
switch checksums.Algorithm {
@@ -1970,6 +2043,26 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
gotSum = input.ChecksumCRC64NVME
checksums.CRC64NVME = &value
crc64nvme = &value
case types.ChecksumAlgorithmSha512:
gotSum = input.ChecksumSHA512
checksums.SHA512 = &value
sha512 = &value
case types.ChecksumAlgorithmMd5:
gotSum = input.ChecksumMD5
checksums.MD5 = &value
md5sum = &value
case types.ChecksumAlgorithmXxhash64:
gotSum = input.ChecksumXXHASH64
checksums.XXHASH64 = &value
xxhash64 = &value
case types.ChecksumAlgorithmXxhash3:
gotSum = input.ChecksumXXHASH3
checksums.XXHASH3 = &value
xxhash3 = &value
case types.ChecksumAlgorithmXxhash128:
gotSum = input.ChecksumXXHASH128
checksums.XXHASH128 = &value
xxhash128 = &value
}
// Check if the provided checksum and the calculated one are the same.
@@ -2176,6 +2269,11 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
ChecksumSHA1: sha1,
ChecksumSHA256: sha256,
ChecksumCRC64NVME: crc64nvme,
ChecksumSHA512: sha512,
ChecksumMD5: md5sum,
ChecksumXXHASH64: xxhash64,
ChecksumXXHASH3: xxhash3,
ChecksumXXHASH128: xxhash128,
ChecksumType: &checksums.Type,
}, versionID, nil
}
@@ -2212,6 +2310,11 @@ func validatePartChecksum(checksum s3response.Checksum, part types.CompletedPart
{part.ChecksumSHA1, getString(checksum.SHA1), types.ChecksumAlgorithmSha1},
{part.ChecksumSHA256, getString(checksum.SHA256), types.ChecksumAlgorithmSha256},
{part.ChecksumCRC64NVME, getString(checksum.CRC64NVME), types.ChecksumAlgorithmCrc64nvme},
{part.ChecksumSHA512, getString(checksum.SHA512), types.ChecksumAlgorithmSha512},
{part.ChecksumMD5, getString(checksum.MD5), types.ChecksumAlgorithmMd5},
{part.ChecksumXXHASH64, getString(checksum.XXHASH64), types.ChecksumAlgorithmXxhash64},
{part.ChecksumXXHASH3, getString(checksum.XXHASH3), types.ChecksumAlgorithmXxhash3},
{part.ChecksumXXHASH128, getString(checksum.XXHASH128), types.ChecksumAlgorithmXxhash128},
} {
if cs.checksum == nil {
continue
@@ -2254,6 +2357,21 @@ func numberOfChecksums(part types.CompletedPart) int {
if getString(part.ChecksumCRC64NVME) != "" {
counter++
}
if getString(part.ChecksumSHA512) != "" {
counter++
}
if getString(part.ChecksumMD5) != "" {
counter++
}
if getString(part.ChecksumXXHASH64) != "" {
counter++
}
if getString(part.ChecksumXXHASH3) != "" {
counter++
}
if getString(part.ChecksumXXHASH128) != "" {
counter++
}
return counter
}
@@ -2778,6 +2896,11 @@ func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3resp
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
ChecksumCRC64NVME: checksum.CRC64NVME,
ChecksumSHA512: checksum.SHA512,
ChecksumMD5: checksum.MD5,
ChecksumXXHASH64: checksum.XXHASH64,
ChecksumXXHASH3: checksum.XXHASH3,
ChecksumXXHASH128: checksum.XXHASH128,
})
}
@@ -2899,6 +3022,11 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart
{input.ChecksumSHA1, utils.HashTypeSha1},
{input.ChecksumSHA256, utils.HashTypeSha256},
{input.ChecksumCRC64NVME, utils.HashTypeCRC64NVME},
{input.ChecksumSHA512, utils.HashTypeSha512},
{input.ChecksumMD5, utils.HashTypeMd5},
{input.ChecksumXXHASH64, utils.HashTypeXXHASH64},
{input.ChecksumXXHASH3, utils.HashTypeXXHASH3},
{input.ChecksumXXHASH128, utils.HashTypeXXHASH128},
}
for _, config := range hashConfigs {
@@ -3024,24 +3152,8 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart
sum = hashRdr.Sum()
}
// Assign the checksum
switch checksums.Algorithm {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = &sum
res.ChecksumCRC32 = &sum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = &sum
res.ChecksumCRC32C = &sum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = &sum
res.ChecksumSHA1 = &sum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = &sum
res.ChecksumSHA256 = &sum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = &sum
res.ChecksumCRC64NVME = &sum
}
setStoredChecksum(&checksum, checksums.Algorithm, &sum)
setUploadPartChecksum(res, checksums.Algorithm, &sum)
err := p.storeChecksums(f.File(), bucket, partPath, checksum)
if err != nil {
@@ -3083,6 +3195,16 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart
res.ChecksumSHA256 = &sumToReturn
case utils.HashTypeCRC64NVME:
res.ChecksumCRC64NVME = &sumToReturn
case utils.HashTypeSha512:
res.ChecksumSHA512 = &sumToReturn
case utils.HashTypeMd5:
res.ChecksumMD5 = &sumToReturn
case utils.HashTypeXXHASH64:
res.ChecksumXXHASH64 = &sumToReturn
case utils.HashTypeXXHASH3:
res.ChecksumXXHASH3 = &sumToReturn
case utils.HashTypeXXHASH128:
res.ChecksumXXHASH128 = &sumToReturn
}
}
}
@@ -3321,18 +3443,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
}
sum := hashRdr.Sum()
switch algo {
case types.ChecksumAlgorithmCrc32:
checksums.CRC32 = &sum
case types.ChecksumAlgorithmCrc32c:
checksums.CRC32C = &sum
case types.ChecksumAlgorithmSha1:
checksums.SHA1 = &sum
case types.ChecksumAlgorithmSha256:
checksums.SHA256 = &sum
case types.ChecksumAlgorithmCrc64nvme:
checksums.CRC64NVME = &sum
}
setStoredChecksum(&checksums, algo, &sum)
err := p.storeChecksums(f.File(), *upi.Bucket, partPath, checksums)
if err != nil {
@@ -3374,6 +3485,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumSHA512: checksums.SHA512,
ChecksumMD5: checksums.MD5,
ChecksumXXHASH64: checksums.XXHASH64,
ChecksumXXHASH3: checksums.XXHASH3,
ChecksumXXHASH128: checksums.XXHASH128,
}, nil
}
@@ -3389,6 +3505,16 @@ func getEmptyChecksumValue(algo types.ChecksumAlgorithm) string {
return "2jmj7l5rSw0yVb/vlWAYkK/YBwk="
case types.ChecksumAlgorithmSha256:
return "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
case types.ChecksumAlgorithmSha512:
return "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="
case types.ChecksumAlgorithmMd5:
return "1B2M2Y8AsgTpgAmY7PhCfg=="
case types.ChecksumAlgorithmXxhash64:
return "70bbN1HY6Zk="
case types.ChecksumAlgorithmXxhash3:
return "LQaABTjTlMI="
case types.ChecksumAlgorithmXxhash128:
return "maoG0wFHmNhgAcMkRo1Jfw=="
default:
// default to crc64nvme
return "AAAAAAAAAAA="
@@ -3464,6 +3590,11 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
{po.ChecksumSHA1, utils.HashTypeSha1},
{po.ChecksumSHA256, utils.HashTypeSha256},
{po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME},
{po.ChecksumSHA512, utils.HashTypeSha512},
{po.ChecksumMD5, utils.HashTypeMd5},
{po.ChecksumXXHASH64, utils.HashTypeXXHASH64},
{po.ChecksumXXHASH3, utils.HashTypeXXHASH3},
{po.ChecksumXXHASH128, utils.HashTypeXXHASH128},
}
for _, config := range hashConfigs {
@@ -3535,19 +3666,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
Algorithm: checksumAlgorithm,
}
// Store the calculated checksum in the object metadata
switch checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = &expectedSum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = &expectedSum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = &expectedSum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = &expectedSum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = &expectedSum
}
setStoredChecksum(&checksum, checksumAlgorithm, &expectedSum)
err = p.storeChecksums(nil, *po.Bucket, *po.Key, checksum)
if err != nil {
@@ -3564,6 +3683,11 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
ChecksumCRC64NVME: checksum.CRC64NVME,
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
ChecksumSHA512: checksum.SHA512,
ChecksumMD5: checksum.MD5,
ChecksumXXHASH64: checksum.XXHASH64,
ChecksumXXHASH3: checksum.XXHASH3,
ChecksumXXHASH128: checksum.XXHASH128,
}, nil
}
@@ -3709,19 +3833,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
Algorithm: checksumAlgorithm,
}
// Store the calculated checksum in the object metadata
switch checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = &sum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = &sum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = &sum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = &sum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = &sum
}
setStoredChecksum(&checksum, checksumAlgorithm, &sum)
err = p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
@@ -3822,6 +3934,11 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
ChecksumCRC64NVME: checksum.CRC64NVME,
ChecksumSHA512: checksum.SHA512,
ChecksumMD5: checksum.MD5,
ChecksumXXHASH64: checksum.XXHASH64,
ChecksumXXHASH3: checksum.XXHASH3,
ChecksumXXHASH128: checksum.XXHASH128,
Size: &objsize,
ChecksumType: checksum.Type,
}, nil
@@ -4423,6 +4540,11 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumSHA512: checksums.SHA512,
ChecksumMD5: checksums.MD5,
ChecksumXXHASH64: checksums.XXHASH64,
ChecksumXXHASH3: checksums.XXHASH3,
ChecksumXXHASH128: checksums.XXHASH128,
ChecksumType: checksums.Type,
AcceptRanges: backend.GetPtrFromString("bytes"),
ContentLength: &length,
@@ -4586,6 +4708,11 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumSHA512: checksums.SHA512,
ChecksumMD5: checksums.MD5,
ChecksumXXHASH64: checksums.XXHASH64,
ChecksumXXHASH3: checksums.XXHASH3,
ChecksumXXHASH128: checksums.XXHASH128,
ChecksumType: checksums.Type,
PartsCount: partsCount,
}, nil
@@ -4837,6 +4964,11 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumSHA512: checksums.SHA512,
ChecksumMD5: checksums.MD5,
ChecksumXXHASH64: checksums.XXHASH64,
ChecksumXXHASH3: checksums.XXHASH3,
ChecksumXXHASH128: checksums.XXHASH128,
ChecksumType: checksums.Type,
TagCount: tagCount,
PartsCount: partsCount,
@@ -4880,6 +5012,11 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr
ChecksumSHA1: data.ChecksumSHA1,
ChecksumSHA256: data.ChecksumSHA256,
ChecksumCRC64NVME: data.ChecksumCRC64NVME,
ChecksumSHA512: data.ChecksumSHA512,
ChecksumMD5: data.ChecksumMD5,
ChecksumXXHASH64: data.ChecksumXXHASH64,
ChecksumXXHASH3: data.ChecksumXXHASH3,
ChecksumXXHASH128: data.ChecksumXXHASH128,
ChecksumType: data.ChecksumType,
},
}, nil
@@ -5026,6 +5163,11 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
var sha1 *string
var sha256 *string
var crc64nvme *string
var sha512 *string
var md5sum *string
var xxhash64 *string
var xxhash3 *string
var xxhash128 *string
var chType types.ChecksumType
dstObjdPath := joinPathWithTrailer(dstBucket, dstObject)
@@ -5086,6 +5228,21 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
case utils.HashTypeCRC64NVME:
checksums.CRC64NVME = &sum
crc64nvme = &sum
case utils.HashTypeSha512:
checksums.SHA512 = &sum
sha512 = &sum
case utils.HashTypeMd5:
checksums.MD5 = &sum
md5sum = &sum
case utils.HashTypeXXHASH64:
checksums.XXHASH64 = &sum
xxhash64 = &sum
case utils.HashTypeXXHASH3:
checksums.XXHASH3 = &sum
xxhash3 = &sum
case utils.HashTypeXXHASH128:
checksums.XXHASH128 = &sum
xxhash128 = &sum
}
// If a new checksum is calculated, the checksum type
@@ -5209,6 +5366,11 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
sha1 = res.ChecksumSHA1
sha256 = res.ChecksumSHA256
crc64nvme = res.ChecksumCRC64NVME
sha512 = res.ChecksumSHA512
md5sum = res.ChecksumMD5
xxhash64 = res.ChecksumXXHASH64
xxhash3 = res.ChecksumXXHASH3
xxhash128 = res.ChecksumXXHASH128
chType = res.ChecksumType
}
@@ -5226,6 +5388,11 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
ChecksumSHA1: sha1,
ChecksumSHA256: sha256,
ChecksumCRC64NVME: crc64nvme,
ChecksumSHA512: sha512,
ChecksumMD5: md5sum,
ChecksumXXHASH64: xxhash64,
ChecksumXXHASH3: xxhash3,
ChecksumXXHASH128: xxhash128,
ChecksumType: chType,
},
VersionId: version,
backend/s3proxy/s3.go
+75
View File
@@ -483,6 +483,21 @@ func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
input.ChecksumSHA256 = nil
}
if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" {
input.ChecksumSHA512 = nil
}
if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" {
input.ChecksumMD5 = nil
}
if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" {
input.ChecksumXXHASH64 = nil
}
if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" {
input.ChecksumXXHASH3 = nil
}
if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" {
input.ChecksumXXHASH128 = nil
}
if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
input.ExpectedBucketOwner = nil
}
@@ -518,6 +533,11 @@ func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
ChecksumCRC64NVME: out.ChecksumCRC64NVME,
ChecksumSHA1: out.ChecksumSHA1,
ChecksumSHA256: out.ChecksumSHA256,
ChecksumSHA512: out.ChecksumSHA512,
ChecksumMD5: out.ChecksumMD5,
ChecksumXXHASH64: out.ChecksumXXHASH64,
ChecksumXXHASH3: out.ChecksumXXHASH3,
ChecksumXXHASH128: out.ChecksumXXHASH128,
ChecksumType: &out.ChecksumType,
}
if out.VersionId != nil {
@@ -653,6 +673,11 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
ChecksumCRC64NVME: p.ChecksumCRC64NVME,
ChecksumSHA1: p.ChecksumSHA1,
ChecksumSHA256: p.ChecksumSHA256,
ChecksumSHA512: p.ChecksumSHA512,
ChecksumMD5: p.ChecksumMD5,
ChecksumXXHASH64: p.ChecksumXXHASH64,
ChecksumXXHASH3: p.ChecksumXXHASH3,
ChecksumXXHASH128: p.ChecksumXXHASH128,
})
}
pnm, err := strconv.Atoi(*output.PartNumberMarker)
@@ -709,6 +734,21 @@ func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
input.ChecksumSHA256 = nil
}
if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" {
input.ChecksumSHA512 = nil
}
if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" {
input.ChecksumMD5 = nil
}
if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" {
input.ChecksumXXHASH64 = nil
}
if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" {
input.ChecksumXXHASH3 = nil
}
if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" {
input.ChecksumXXHASH128 = nil
}
if input.ContentMD5 != nil && *input.ContentMD5 == "" {
input.ContentMD5 = nil
}
@@ -790,6 +830,11 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME,
ChecksumSHA1: output.CopyPartResult.ChecksumSHA1,
ChecksumSHA256: output.CopyPartResult.ChecksumSHA256,
ChecksumSHA512: output.CopyPartResult.ChecksumSHA512,
ChecksumMD5: output.CopyPartResult.ChecksumMD5,
ChecksumXXHASH64: output.CopyPartResult.ChecksumXXHASH64,
ChecksumXXHASH3: output.CopyPartResult.ChecksumXXHASH3,
ChecksumXXHASH128: output.CopyPartResult.ChecksumXXHASH128,
}, nil
}
@@ -815,6 +860,21 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
input.ChecksumSHA256 = nil
}
if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" {
input.ChecksumSHA512 = nil
}
if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" {
input.ChecksumMD5 = nil
}
if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" {
input.ChecksumXXHASH64 = nil
}
if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" {
input.ChecksumXXHASH3 = nil
}
if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" {
input.ChecksumXXHASH128 = nil
}
if input.ContentDisposition != nil && *input.ContentDisposition == "" {
input.ContentDisposition = nil
}
@@ -913,6 +973,11 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
ChecksumSHA1: input.ChecksumSHA1,
ChecksumSHA256: input.ChecksumSHA256,
ChecksumCRC64NVME: input.ChecksumCRC64NVME,
ChecksumSHA512: input.ChecksumSHA512,
ChecksumMD5: input.ChecksumMD5,
ChecksumXXHASH64: input.ChecksumXXHASH64,
ChecksumXXHASH3: input.ChecksumXXHASH3,
ChecksumXXHASH128: input.ChecksumXXHASH128,
ContentMD5: input.ContentMD5,
ExpectedBucketOwner: input.ExpectedBucketOwner,
GrantFullControl: input.GrantFullControl,
@@ -947,6 +1012,11 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
ChecksumCRC64NVME: output.ChecksumCRC64NVME,
ChecksumSHA1: output.ChecksumSHA1,
ChecksumSHA256: output.ChecksumSHA256,
ChecksumSHA512: output.ChecksumSHA512,
ChecksumMD5: output.ChecksumMD5,
ChecksumXXHASH64: output.ChecksumXXHASH64,
ChecksumXXHASH3: output.ChecksumXXHASH3,
ChecksumXXHASH128: output.ChecksumXXHASH128,
Size: output.Size,
}, nil
}
@@ -1292,6 +1362,11 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input s3response.CopyObjectInp
ChecksumCRC64NVME: out.CopyObjectResult.ChecksumCRC64NVME,
ChecksumSHA1: out.CopyObjectResult.ChecksumSHA1,
ChecksumSHA256: out.CopyObjectResult.ChecksumSHA256,
ChecksumSHA512: out.CopyObjectResult.ChecksumSHA512,
ChecksumMD5: out.CopyObjectResult.ChecksumMD5,
ChecksumXXHASH64: out.CopyObjectResult.ChecksumXXHASH64,
ChecksumXXHASH3: out.CopyObjectResult.ChecksumXXHASH3,
ChecksumXXHASH128: out.CopyObjectResult.ChecksumXXHASH128,
ChecksumType: out.CopyObjectResult.ChecksumType,
ETag: out.CopyObjectResult.ETag,
LastModified: out.CopyObjectResult.LastModified,
go.mod
+2
View File
@@ -13,6 +13,7 @@ require (
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.18
github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0
github.com/aws/smithy-go v1.25.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/davecgh/go-spew v1.1.1
github.com/go-ldap/ldap/v3 v3.4.13
github.com/gofiber/fiber/v2 v2.52.13
@@ -30,6 +31,7 @@ require (
github.com/urfave/cli/v2 v2.27.7
github.com/valyala/fasthttp v1.70.0
github.com/versity/scoutfs-go v0.0.0-20240625221833-95fd765b760b
github.com/zeebo/xxh3 v1.1.0
golang.org/x/sync v0.20.0
golang.org/x/sys v0.43.0
gopkg.in/yaml.v3 v3.0.1
go.sum
+6
View File
@@ -63,6 +63,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.0 h1:ks8KBcZPh3PYISr5dAiXCM5/Thcu
github.com/aws/aws-sdk-go-v2/service/sts v1.42.0/go.mod h1:pFw33T0WLvXU3rw1WBkpMlkgIn54eCB5FYLhjDc9Foo=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
@@ -194,6 +196,10 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
s3api/controllers/bucket-post.go
+10
View File
@@ -222,6 +222,11 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
})
if err != nil {
return &Response{
@@ -285,6 +290,11 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
"x-amz-version-id": utils.GetStringPtr(res.VersionID),
},
s3api/controllers/bucket-post_test.go
+5
View File
@@ -547,6 +547,11 @@ func TestS3ApiController_POSTObject(t *testing.T) {
"x-amz-checksum-crc64nvme": utils.GetStringPtr(""),
"x-amz-checksum-sha1": utils.GetStringPtr(""),
"x-amz-checksum-sha256": utils.GetStringPtr(""),
"x-amz-checksum-sha512": utils.GetStringPtr(""),
"x-amz-checksum-md5": utils.GetStringPtr(""),
"x-amz-checksum-xxhash64": utils.GetStringPtr(""),
"x-amz-checksum-xxhash3": utils.GetStringPtr(""),
"x-amz-checksum-xxhash128": utils.GetStringPtr(""),
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
"x-amz-version-id": utils.GetStringPtr("vid-123"),
},
s3api/controllers/object-get.go
+5
View File
@@ -556,6 +556,11 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"Content-Type": utils.ApplyOverride(res.ContentType, responseOverrides["Content-Type"]),
"x-amz-version-id": res.VersionId,
"Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength),
s3api/controllers/object-get_test.go
+10
View File
@@ -799,6 +799,11 @@ func TestS3ApiController_GetObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
@@ -844,6 +849,11 @@ func TestS3ApiController_GetObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
s3api/controllers/object-head.go
+5
View File
@@ -188,6 +188,11 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-version-id": res.VersionId,
"x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount),
"x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode),
s3api/controllers/object-head_test.go
+10
View File
@@ -189,6 +189,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
@@ -233,6 +238,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
s3api/controllers/object-post.go
+5
View File
@@ -375,6 +375,11 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
ChecksumType: checksumType,
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
s3api/controllers/object-put.go
+20
View File
@@ -306,6 +306,11 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
})
var headers map[string]*string
if err == nil {
@@ -316,6 +321,11 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
}
}
return &Response{
@@ -789,6 +799,11 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
})
@@ -800,6 +815,11 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
"x-amz-version-id": &res.VersionID,
"x-amz-object-size": utils.ConvertPtrToStringPtr(res.Size),
s3api/controllers/object-put_test.go
+20
View File
@@ -535,6 +535,11 @@ func TestS3ApiController_UploadPart(t *testing.T) {
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
@@ -1214,6 +1219,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-checksum-type": nil,
"x-amz-version-id": emptyStringPtr,
"x-amz-object-size": nil,
@@ -1255,6 +1265,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
ChecksumSHA1: utils.GetStringPtr("sha1"),
ChecksumSHA256: utils.GetStringPtr("sha256"),
ChecksumCRC64NVME: utils.GetStringPtr("crc64nvme"),
ChecksumSHA512: utils.GetStringPtr("sha512"),
ChecksumMD5: utils.GetStringPtr("md5"),
ChecksumXXHASH64: utils.GetStringPtr("xxhash64"),
ChecksumXXHASH3: utils.GetStringPtr("xxhash3"),
ChecksumXXHASH128: utils.GetStringPtr("xxhash128"),
ChecksumType: types.ChecksumTypeComposite,
VersionID: "versionId",
Size: &objSize,
@@ -1269,6 +1284,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
"x-amz-checksum-crc64nvme": utils.GetStringPtr("crc64nvme"),
"x-amz-checksum-sha1": utils.GetStringPtr("sha1"),
"x-amz-checksum-sha256": utils.GetStringPtr("sha256"),
"x-amz-checksum-sha512": utils.GetStringPtr("sha512"),
"x-amz-checksum-md5": utils.GetStringPtr("md5"),
"x-amz-checksum-xxhash64": utils.GetStringPtr("xxhash64"),
"x-amz-checksum-xxhash3": utils.GetStringPtr("xxhash3"),
"x-amz-checksum-xxhash128": utils.GetStringPtr("xxhash128"),
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
"x-amz-version-id": utils.GetStringPtr("versionId"),
"x-amz-object-size": utils.ConvertToStringPtr(objSize),
s3api/middlewares/checksum.go
+2 -2
View File
@@ -49,7 +49,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
var err error
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
r, err = utils.NewHashReader(r, md5sum, utils.HashTypeMd5)
r, err = utils.NewHashReader(r, md5sum, utils.HashTypeContentMD5)
return r
})
if err != nil {
@@ -70,7 +70,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
return s3err.GetAPIError(s3err.ErrInvalidDigest)
}
rdr, err = utils.NewHashReader(bytes.NewReader(body), md5sum, utils.HashTypeMd5)
rdr, err = utils.NewHashReader(bytes.NewReader(body), md5sum, utils.HashTypeContentMD5)
if err != nil {
return err
}
s3api/utils/chunk-reader.go
+11 -1
View File
@@ -79,6 +79,11 @@ const (
checksumTypeSha1 checksumType = "x-amz-checksum-sha1"
checksumTypeSha256 checksumType = "x-amz-checksum-sha256"
checksumTypeCrc64nvme checksumType = "x-amz-checksum-crc64nvme"
checksumTypeSha512 checksumType = "x-amz-checksum-sha512"
checksumTypeMd5 checksumType = "x-amz-checksum-md5"
checksumTypeXxhash64 checksumType = "x-amz-checksum-xxhash64"
checksumTypeXxhash3 checksumType = "x-amz-checksum-xxhash3"
checksumTypeXxhash128 checksumType = "x-amz-checksum-xxhash128"
)
func (c checksumType) isValid() bool {
@@ -86,7 +91,12 @@ func (c checksumType) isValid() bool {
c == checksumTypeCrc32c ||
c == checksumTypeSha1 ||
c == checksumTypeSha256 ||
c == checksumTypeCrc64nvme
c == checksumTypeCrc64nvme ||
c == checksumTypeSha512 ||
c == checksumTypeMd5 ||
c == checksumTypeXxhash64 ||
c == checksumTypeXxhash3 ||
c == checksumTypeXxhash128
}
// Extracts and validates the checksum type from the 'X-Amz-Trailer' header
s3api/utils/csum-reader.go
+75 -6
View File
@@ -18,6 +18,7 @@ import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"errors"
@@ -28,14 +29,18 @@ import (
"io"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
)
// HashType identifies the checksum algorithm to be used
type HashType string
const (
// HashTypeMd5 generates MD5 checksum for the data stream
// HashTypeContentMD5 generates MD5 checksum for the Content-MD5 header.
HashTypeContentMD5 HashType = "content-md5"
// HashTypeMd5 generates MD5 Base64-Encoded checksum for the data stream
HashTypeMd5 HashType = "md5"
// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
HashTypeSha256 HashType = "sha256"
@@ -43,12 +48,20 @@ const (
HashTypeSha256Hex HashType = "sha256-hex"
// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
HashTypeSha1 HashType = "sha1"
// HashTypeSha512 generates SHA512 Base64-Encoded checksum for the data stream
HashTypeSha512 HashType = "sha512"
// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
HashTypeCRC32 HashType = "crc32"
// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
HashTypeCRC32C HashType = "crc32c"
// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
HashTypeCRC64NVME HashType = "crc64nvme"
// HashTypeXXHASH64 generates XXHASH64 Base64-Encoded checksum for the data stream
HashTypeXXHASH64 HashType = "xxhash64"
// HashTypeXXHASH3 generates XXHASH3 Base64-Encoded checksum for the data stream
HashTypeXXHASH3 HashType = "xxhash3"
// HashTypeXXHASH128 generates XXHASH128 Base64-Encoded checksum for the data stream
HashTypeXXHASH128 HashType = "xxhash128"
// HashTypeNone is a no-op checksum for the data stream
HashTypeNone HashType = "none"
)
@@ -75,7 +88,7 @@ var (
func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) {
var hash hash.Hash
switch ht {
case HashTypeMd5:
case HashTypeContentMD5, HashTypeMd5:
hash = md5.New()
case HashTypeSha256Hex:
hash = sha256.New()
@@ -83,12 +96,20 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
hash = sha256.New()
case HashTypeSha1:
hash = sha1.New()
case HashTypeSha512:
hash = sha512.New()
case HashTypeCRC32:
hash = crc32.NewIEEE()
case HashTypeCRC32C:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeCRC64NVME:
hash = crc64.New(crc64NVMETable)
case HashTypeXXHASH64:
hash = xxhash.New()
case HashTypeXXHASH3:
hash = xxh3.New()
case HashTypeXXHASH128:
hash = xxh3.New128()
case HashTypeNone:
hash = noop{}
default:
@@ -112,11 +133,16 @@ func (hr *HashReader) Read(p []byte) (int, error) {
}
if errors.Is(readerr, io.EOF) && hr.sum != "" {
switch hr.hashType {
case HashTypeMd5:
case HashTypeContentMD5:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrBadDigest)
}
case HashTypeMd5:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmMd5)
}
case HashTypeSha256Hex:
sum := hr.Sum()
if sum != hr.sum {
@@ -142,11 +168,31 @@ func (hr *HashReader) Read(p []byte) (int, error) {
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
}
case HashTypeSha512:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha512)
}
case HashTypeCRC64NVME:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
}
case HashTypeXXHASH64:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash64)
}
case HashTypeXXHASH3:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash3)
}
case HashTypeXXHASH128:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash128)
}
default:
return n, errInvalidHashType
}
@@ -161,7 +207,7 @@ func (hr *HashReader) SetReader(r io.Reader) {
// Sum returns the checksum hash of the data read so far
func (hr *HashReader) Sum() string {
switch hr.hashType {
case HashTypeMd5:
case HashTypeContentMD5, HashTypeMd5:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256Hex:
return hex.EncodeToString(hr.hash.Sum(nil))
@@ -173,8 +219,16 @@ func (hr *HashReader) Sum() string {
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha512:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeCRC64NVME:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH64:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH3:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH128:
return Base64SumString(hr.hash.Sum(nil))
default:
return ""
}
@@ -197,8 +251,8 @@ func (n noop) Reset() {}
func (n noop) Size() int { return 0 }
func (n noop) BlockSize() int { return 1 }
// IsChecksumComposable tests if the final foll object crc can be calculated
// based on the part crc values.
// IsChecksumComposable tests if the final full-object crc can be calculated
// from the part crc values.
func IsChecksumComposable(algo types.ChecksumAlgorithm) bool {
switch algo {
case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c, types.ChecksumAlgorithmCrc64nvme:
@@ -309,17 +363,32 @@ func AddCRCChecksum(algo types.ChecksumAlgorithm, crc, partCrc string, partLen i
// - CRC32C
// - SHA1
// - SHA256
// - SHA512
// - MD5
// - XXHASH64
// - XXHASH3
// - XXHASH128
func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
var hasher hash.Hash
switch ht {
case HashTypeMd5:
hasher = md5.New()
case HashTypeSha256:
hasher = sha256.New()
case HashTypeSha1:
hasher = sha1.New()
case HashTypeSha512:
hasher = sha512.New()
case HashTypeCRC32:
hasher = crc32.NewIEEE()
case HashTypeCRC32C:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeXXHASH64:
hasher = xxhash.New()
case HashTypeXXHASH3:
hasher = xxh3.New()
case HashTypeXXHASH128:
hasher = xxh3.New128()
case HashTypeNone:
hasher = noop{}
default:
s3api/utils/csum-reader_test.go
+99
View File
@@ -15,12 +15,22 @@
package utils
import (
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/stretchr/testify/assert"
"github.com/zeebo/xxh3"
)
func TestAddCRCChecksum_CRC32(t *testing.T) {
@@ -118,3 +128,92 @@ func TestAddCRCChecksum_CRC64NVME(t *testing.T) {
t.Errorf("CRC64NVME combine mismatch: got %x, want %x", combinedVal, crcFull)
}
}
func base64HashForTest(h hash.Hash, data []byte) string {
h.Write(data)
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func TestNewHashReader_NewChecksumAlgorithms(t *testing.T) {
data := []byte("checksum payload")
tests := []struct {
name string
hashType HashType
hasher hash.Hash
}{
{name: "md5", hashType: HashTypeMd5, hasher: md5.New()},
{name: "sha1", hashType: HashTypeSha1, hasher: sha1.New()},
{name: "sha256", hashType: HashTypeSha256, hasher: sha256.New()},
{name: "sha512", hashType: HashTypeSha512, hasher: sha512.New()},
{name: "crc32", hashType: HashTypeCRC32, hasher: crc32.NewIEEE()},
{name: "crc32c", hashType: HashTypeCRC32C, hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
{name: "crc64nvme", hashType: HashTypeCRC64NVME, hasher: crc64.New(crc64NVMETable)},
{name: "xxhash64", hashType: HashTypeXXHASH64, hasher: xxhash.New()},
{name: "xxhash3", hashType: HashTypeXXHASH3, hasher: xxh3.New()},
{name: "xxhash128", hashType: HashTypeXXHASH128, hasher: xxh3.New128()},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
expected := base64HashForTest(tt.hasher, data)
rdr, err := NewHashReader(bytes.NewReader(data), expected, tt.hashType)
if !assert.NoError(t, err) {
return
}
_, err = io.Copy(io.Discard, rdr)
assert.NoError(t, err)
assert.Equal(t, expected, rdr.Sum())
})
}
}
func TestNewCompositeChecksumReader_NewChecksumAlgorithms(t *testing.T) {
part1 := []byte("part one")
part2 := []byte("part two")
tests := []struct {
name string
hashType HashType
newHasher func() hash.Hash
}{
{name: "md5", hashType: HashTypeMd5, newHasher: md5.New},
{name: "sha1", hashType: HashTypeSha1, newHasher: sha1.New},
{name: "sha256", hashType: HashTypeSha256, newHasher: sha256.New},
{name: "sha512", hashType: HashTypeSha512, newHasher: sha512.New},
{name: "crc32", hashType: HashTypeCRC32, newHasher: func() hash.Hash { return crc32.NewIEEE() }},
{name: "crc32c", hashType: HashTypeCRC32C, newHasher: func() hash.Hash { return crc32.New(crc32.MakeTable(crc32.Castagnoli)) }},
{name: "xxhash64", hashType: HashTypeXXHASH64, newHasher: func() hash.Hash { return xxhash.New() }},
{name: "xxhash3", hashType: HashTypeXXHASH3, newHasher: func() hash.Hash { return xxh3.New() }},
{name: "xxhash128", hashType: HashTypeXXHASH128, newHasher: func() hash.Hash { return xxh3.New128() }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
part1Sum := base64HashForTest(tt.newHasher(), part1)
part2Sum := base64HashForTest(tt.newHasher(), part2)
composite, err := NewCompositeChecksumReader(tt.hashType)
if !assert.NoError(t, err) {
return
}
assert.NoError(t, composite.Process(part1Sum))
assert.NoError(t, composite.Process(part2Sum))
part1Bytes, err := base64.StdEncoding.DecodeString(part1Sum)
if !assert.NoError(t, err) {
return
}
part2Bytes, err := base64.StdEncoding.DecodeString(part2Sum)
if !assert.NoError(t, err) {
return
}
expectedHasher := tt.newHasher()
expectedHasher.Write(part1Bytes)
expectedHasher.Write(part2Bytes)
expected := base64.StdEncoding.EncodeToString(expectedHasher.Sum(nil))
assert.Equal(t, expected, composite.Sum())
})
}
}
s3api/utils/unsigned-chunk-reader.go
+14
View File
@@ -17,8 +17,10 @@ package utils
import (
"bufio"
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"errors"
"hash"
@@ -30,8 +32,10 @@ import (
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
)
var (
@@ -353,6 +357,16 @@ func getHasher(ct checksumType) (hash.Hash, error) {
return sha1.New(), nil
case checksumTypeSha256:
return sha256.New(), nil
case checksumTypeSha512:
return sha512.New(), nil
case checksumTypeMd5:
return md5.New(), nil
case checksumTypeXxhash64:
return xxhash.New(), nil
case checksumTypeXxhash3:
return xxh3.New(), nil
case checksumTypeXxhash128:
return xxh3.New128(), nil
default:
return nil, errors.New("unsupported checksum type")
}
s3api/utils/utils.go
+31 -1
View File
@@ -721,8 +721,13 @@ var checksumLengths = map[types.ChecksumAlgorithm]int{
types.ChecksumAlgorithmCrc32: 4,
types.ChecksumAlgorithmCrc32c: 4,
types.ChecksumAlgorithmCrc64nvme: 8,
types.ChecksumAlgorithmMd5: 16,
types.ChecksumAlgorithmSha1: 20,
types.ChecksumAlgorithmSha256: 32,
types.ChecksumAlgorithmSha512: 64,
types.ChecksumAlgorithmXxhash64: 8,
types.ChecksumAlgorithmXxhash3: 8,
types.ChecksumAlgorithmXxhash128: 16,
}
func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
@@ -753,7 +758,12 @@ func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
alg != types.ChecksumAlgorithmCrc32c &&
alg != types.ChecksumAlgorithmSha1 &&
alg != types.ChecksumAlgorithmSha256 &&
alg != types.ChecksumAlgorithmCrc64nvme {
alg != types.ChecksumAlgorithmCrc64nvme &&
alg != types.ChecksumAlgorithmSha512 &&
alg != types.ChecksumAlgorithmMd5 &&
alg != types.ChecksumAlgorithmXxhash64 &&
alg != types.ChecksumAlgorithmXxhash3 &&
alg != types.ChecksumAlgorithmXxhash128 {
debuglogger.Logf("invalid checksum algorithm: %v\n", alg)
return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
}
@@ -799,6 +809,26 @@ var checksumMap checksumSchema = checksumSchema{
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmSha512: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmMd5: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash64: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash3: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash128: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
// Both could be empty
"": checksumTypeSchema{
"": struct{}{},
s3api/utils/utils_test.go
+211
View File
@@ -707,6 +707,41 @@ func TestIsChecksumAlgorithmValid(t *testing.T) {
},
wantErr: false,
},
{
name: "sha512",
args: args{
alg: types.ChecksumAlgorithmSha512,
},
wantErr: false,
},
{
name: "md5",
args: args{
alg: types.ChecksumAlgorithmMd5,
},
wantErr: false,
},
{
name: "xxhash64",
args: args{
alg: types.ChecksumAlgorithmXxhash64,
},
wantErr: false,
},
{
name: "xxhash3",
args: args{
alg: types.ChecksumAlgorithmXxhash3,
},
wantErr: false,
},
{
name: "xxhash128",
args: args{
alg: types.ChecksumAlgorithmXxhash128,
},
wantErr: false,
},
{
name: "invalid",
args: args{
@@ -806,6 +841,102 @@ func TestIsValidChecksum(t *testing.T) {
},
want: true,
},
{
name: "invalid-crc64nvme",
args: args{
checksum: "ww2FVQ==",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: false,
},
{
name: "valid-crc64nvme",
args: args{
checksum: "AAAAAAAAAAA=",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: true,
},
{
name: "invalid-sha512",
args: args{
checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
algorithm: types.ChecksumAlgorithmSha512,
},
want: false,
},
{
name: "valid-sha512",
args: args{
checksum: "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==",
algorithm: types.ChecksumAlgorithmSha512,
},
want: true,
},
{
name: "invalid-md5",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmMd5,
},
want: false,
},
{
name: "valid-md5",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmMd5,
},
want: true,
},
{
name: "invalid-xxhash64",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: false,
},
{
name: "valid-xxhash64",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: true,
},
{
name: "invalid-xxhash3",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: false,
},
{
name: "valid-xxhash3",
args: args{
checksum: "LQaABTjTlMI=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: true,
},
{
name: "invalid-xxhash128",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: false,
},
{
name: "valid-xxhash128",
args: args{
checksum: "maoG0wFHmNhgAcMkRo1Jfw==",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -1090,6 +1221,86 @@ func Test_checkChecksumTypeAndAlgo(t *testing.T) {
},
wantErr: true,
},
{
name: "full_object-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "composite-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-empty",
args: args{
s3err/s3err.go
+1 -1
View File
@@ -763,7 +763,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrInvalidChecksumAlgorithm: {
Code: "InvalidRequest",
Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]",
Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, CRC64NVME, MD5, SHA1, SHA256, SHA512, XXHASH128, XXHASH3, XXHASH64]",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidChecksumPart: {
s3response/s3response.go
+35
View File
@@ -37,6 +37,11 @@ type PutObjectOutput struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
Size *int64
ChecksumType types.ChecksumType
}
@@ -52,6 +57,11 @@ type Part struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
}
func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -375,6 +385,11 @@ type CopyObjectResult struct {
ChecksumCRC64NVME *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
ChecksumType types.ChecksumType
ETag *string
LastModified *time.Time
@@ -404,6 +419,11 @@ type CopyPartResult struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
// not included in the body
CopySourceVersionId string `xml:"-"`
@@ -439,6 +459,11 @@ type CompleteMultipartUploadResult struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
ChecksumType *types.ChecksumType
}
@@ -559,6 +584,11 @@ type PutObjectInput struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
ContentMD5 *string
ExpectedBucketOwner *string
GrantFullControl *string
@@ -731,6 +761,11 @@ type Checksum struct {
SHA1 *string
SHA256 *string
CRC64NVME *string
SHA512 *string
MD5 *string
XXHASH64 *string
XXHASH3 *string
XXHASH128 *string
}
// LocationConstraint represents the GetBucketLocation response
tests/integration/CompleteMultipartUpload.go
+107 -109
View File
@@ -518,6 +518,26 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error {
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo),
@@ -534,15 +554,7 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error {
cParts := []types.CompletedPart{}
for _, el := range parts {
cParts = append(cParts, types.CompletedPart{
ETag: el.ETag,
PartNumber: el.PartNumber,
ChecksumCRC32: el.ChecksumCRC32C,
ChecksumCRC32C: el.ChecksumCRC32C,
ChecksumSHA1: el.ChecksumSHA1,
ChecksumSHA256: el.ChecksumSHA256,
ChecksumCRC64NVME: el.ChecksumCRC64NVME,
})
cParts = append(cParts, completedPartFromPart(el))
}
mpInput := &s3.CompleteMultipartUploadInput{
@@ -566,6 +578,16 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error {
mpInput.ChecksumSHA256 = getPtr("invalid_sha256")
case types.ChecksumAlgorithmCrc64nvme:
mpInput.ChecksumCRC64NVME = getPtr("invalid_crc64nvme")
case types.ChecksumAlgorithmSha512:
mpInput.ChecksumSHA512 = getPtr("invalid_sha512")
case types.ChecksumAlgorithmMd5:
mpInput.ChecksumMD5 = getPtr("invalid_md5")
case types.ChecksumAlgorithmXxhash64:
mpInput.ChecksumXXHASH64 = getPtr("invalid_xxhash64")
case types.ChecksumAlgorithmXxhash3:
mpInput.ChecksumXXHASH3 = getPtr("invalid_xxhash3")
case types.ChecksumAlgorithmXxhash128:
mpInput.ChecksumXXHASH128 = getPtr("invalid_xxhash128")
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
@@ -608,6 +630,26 @@ func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error {
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo),
withChecksumType(el.t))
@@ -623,34 +665,23 @@ func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error {
cParts := []types.CompletedPart{}
for _, el := range parts {
cParts = append(cParts, types.CompletedPart{
ETag: el.ETag,
PartNumber: el.PartNumber,
ChecksumCRC32: el.ChecksumCRC32,
ChecksumCRC32C: el.ChecksumCRC32C,
ChecksumSHA1: el.ChecksumSHA1,
ChecksumSHA256: el.ChecksumSHA256,
ChecksumCRC64NVME: el.ChecksumCRC64NVME,
})
cParts = append(cParts, completedPartFromPart(el))
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
input := &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: cParts,
},
// Provide one of the parts checksum. In any case
// the final checksum will differ from one of the parts checksum
ChecksumCRC32: cParts[0].ChecksumCRC32,
ChecksumCRC32C: cParts[0].ChecksumCRC32C,
ChecksumSHA1: cParts[0].ChecksumSHA1,
ChecksumSHA256: cParts[0].ChecksumSHA256,
ChecksumCRC64NVME: cParts[0].ChecksumCRC64NVME,
ChecksumType: el.t,
})
ChecksumType: el.t,
}
// Provide one of the part checksums. The final checksum will differ
// from a single part checksum.
setCompleteMultipartUploadChecksum(input, el.algo, getCompletedPartChecksum(cParts[0], el.algo))
_, err = s3client.CompleteMultipartUpload(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
return err
@@ -848,6 +879,11 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
types.ChecksumAlgorithmCrc32c,
types.ChecksumAlgorithmSha1,
types.ChecksumAlgorithmSha256,
types.ChecksumAlgorithmSha512,
types.ChecksumAlgorithmMd5,
types.ChecksumAlgorithmXxhash64,
types.ChecksumAlgorithmXxhash3,
types.ChecksumAlgorithmXxhash128,
} {
mp, err := createMp(s3client, bucket, obj, withChecksumType(types.ChecksumTypeComposite), withChecksum(algo))
if err != nil {
@@ -867,29 +903,13 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
completeParts := make([]types.CompletedPart, 0, len(parts))
for _, part := range parts {
switch algo {
case types.ChecksumAlgorithmCrc32:
err = processCompositeChecksum(hasher, getString(part.ChecksumCRC32))
case types.ChecksumAlgorithmCrc32c:
err = processCompositeChecksum(hasher, getString(part.ChecksumCRC32C))
case types.ChecksumAlgorithmSha1:
err = processCompositeChecksum(hasher, getString(part.ChecksumSHA1))
case types.ChecksumAlgorithmSha256:
err = processCompositeChecksum(hasher, getString(part.ChecksumSHA256))
}
err = processCompositeChecksum(hasher, getString(getPartChecksum(part, algo)))
if err != nil {
return fmt.Errorf("test %v failed: %s", i, err)
}
completeParts = append(completeParts, types.CompletedPart{
ETag: part.ETag,
PartNumber: part.PartNumber,
ChecksumCRC32: part.ChecksumCRC32,
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
})
completeParts = append(completeParts, completedPartFromPart(part))
}
checksum := fmt.Sprintf("%s-%v", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), len(parts))
@@ -903,16 +923,7 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
UploadId: mp.UploadId,
}
switch algo {
case types.ChecksumAlgorithmCrc32:
completeMpInput.ChecksumCRC32 = &checksum
case types.ChecksumAlgorithmCrc32c:
completeMpInput.ChecksumCRC32C = &checksum
case types.ChecksumAlgorithmSha1:
completeMpInput.ChecksumSHA1 = &checksum
case types.ChecksumAlgorithmSha256:
completeMpInput.ChecksumSHA256 = &checksum
}
setCompleteMultipartUploadChecksum(completeMpInput, algo, &checksum)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.CompleteMultipartUpload(ctx, completeMpInput)
@@ -921,17 +932,7 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
return fmt.Errorf("test %v failed: %s", i, err)
}
var gotSum string
switch algo {
case types.ChecksumAlgorithmCrc32:
gotSum = getString(res.ChecksumCRC32)
case types.ChecksumAlgorithmCrc32c:
gotSum = getString(res.ChecksumCRC32C)
case types.ChecksumAlgorithmSha1:
gotSum = getString(res.ChecksumSHA1)
case types.ChecksumAlgorithmSha256:
gotSum = getString(res.ChecksumSHA256)
}
gotSum := getString(getCompleteMultipartUploadChecksum(res, algo))
if gotSum != checksum {
return fmt.Errorf("test %v failed: expected the final checksum to be %s, instead got %s", i, checksum, gotSum)
@@ -947,20 +948,27 @@ func CompleteMultipartUpload_invalid_final_composite_checksum(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for i, test := range []struct {
algo types.ChecksumAlgorithm
crc32 *string
crc32c *string
sha1 *string
sha256 *string
algo types.ChecksumAlgorithm
checksum *string
}{
{types.ChecksumAlgorithmCrc32, getPtr("invalid_checksum"), nil, nil, nil},
{types.ChecksumAlgorithmCrc32, getPtr("ImIEBA==-smth"), nil, nil, nil},
{types.ChecksumAlgorithmCrc32c, nil, getPtr("invalid_checksum"), nil, nil},
{types.ChecksumAlgorithmCrc32c, nil, getPtr("AQIDBA==-12a"), nil, nil},
{types.ChecksumAlgorithmSha1, nil, nil, getPtr("invalid_checksum"), nil},
{types.ChecksumAlgorithmSha1, nil, nil, getPtr("2jmj7l5rSw0yVb/vlWAYkK/YBwk=-10-20"), nil},
{types.ChecksumAlgorithmSha256, nil, nil, nil, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha256, nil, nil, nil, getPtr("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=--3")},
{types.ChecksumAlgorithmCrc32, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmCrc32, getPtr("ImIEBA==-smth")},
{types.ChecksumAlgorithmCrc32c, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmCrc32c, getPtr("AQIDBA==-12a")},
{types.ChecksumAlgorithmSha1, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha1, getPtr("2jmj7l5rSw0yVb/vlWAYkK/YBwk=-10-20")},
{types.ChecksumAlgorithmSha256, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha256, getPtr("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=--3")},
{types.ChecksumAlgorithmSha512, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha512, getPtr("z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==-bad")},
{types.ChecksumAlgorithmMd5, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmMd5, getPtr("1B2M2Y8AsgTpgAmY7PhCfg==-bad")},
{types.ChecksumAlgorithmXxhash64, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmXxhash64, getPtr("70bbN1HY6Zk=-bad")},
{types.ChecksumAlgorithmXxhash3, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmXxhash3, getPtr("LQaABTjTlMI=-bad")},
{types.ChecksumAlgorithmXxhash128, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmXxhash128, getPtr("maoG0wFHmNhgAcMkRo1Jfw==-bad")},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(test.algo), withChecksumType(types.ChecksumTypeComposite))
if err != nil {
@@ -975,29 +983,20 @@ func CompleteMultipartUpload_invalid_final_composite_checksum(s *S3Conf) error {
completeParts := make([]types.CompletedPart, 0, len(parts))
for _, part := range parts {
completeParts = append(completeParts, types.CompletedPart{
ETag: part.ETag,
PartNumber: part.PartNumber,
ChecksumCRC32: part.ChecksumCRC32,
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
})
completeParts = append(completeParts, completedPartFromPart(part))
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
input := &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: completeParts,
},
ChecksumCRC32: test.crc32,
ChecksumCRC32C: test.crc32c,
ChecksumSHA1: test.sha1,
ChecksumSHA256: test.sha256,
})
}
setCompleteMultipartUploadChecksum(input, test.algo, test.checksum)
_, err = s3client.CompleteMultipartUpload(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(test.algo))))); err != nil {
return fmt.Errorf("test %v failed: %w", i, err)
@@ -1090,21 +1089,20 @@ func CompleteMultipartUpload_should_ignore_the_final_checksum(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, insted got %v",
*res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, insted got %v",
*res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, insted got %v",
*res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, insted got %v",
*res.ChecksumSHA256)
for _, algo := range []types.ChecksumAlgorithm{
types.ChecksumAlgorithmCrc32,
types.ChecksumAlgorithmCrc32c,
types.ChecksumAlgorithmSha1,
types.ChecksumAlgorithmSha256,
types.ChecksumAlgorithmSha512,
types.ChecksumAlgorithmMd5,
types.ChecksumAlgorithmXxhash64,
types.ChecksumAlgorithmXxhash3,
types.ChecksumAlgorithmXxhash128,
} {
if checksum := getCompleteMultipartUploadChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, insted got %v", algo, *checksum)
}
}
// If no checksum is specified on mp creation, it should default
// to crc64nvme
tests/integration/CopyObject.go
+14 -22
View File
@@ -1461,17 +1461,13 @@ func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error {
if out.CopyObjectResult.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum")
}
if out.CopyObjectResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected empty crc32c checksum")
}
if out.CopyObjectResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected empty sha1 checksum")
}
if out.CopyObjectResult.ChecksumSHA256 != nil {
return fmt.Errorf("expected empty sha256 checksum")
}
if out.CopyObjectResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected empty crc64nvme checksum")
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if algo == types.ChecksumAlgorithmCrc32 {
continue
}
if checksum := getCopyObjectChecksum(out.CopyObjectResult, algo); checksum != nil {
return fmt.Errorf("expected empty %s checksum", algo)
}
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
@@ -1488,17 +1484,13 @@ func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error {
if res.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum")
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected empty crc32c checksum")
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected empty sha1 checksum")
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected empty sha256 checksum")
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected empty crc64nvme checksum")
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if algo == types.ChecksumAlgorithmCrc32 {
continue
}
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected empty %s checksum", algo)
}
}
return nil
tests/integration/CreateMultipartUpload.go
+10
View File
@@ -390,6 +390,11 @@ func CreateMultipartUpload_type_algo_mismatch(s *S3Conf) error {
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc64nvme},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha256},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha512},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmMd5},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash64},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash3},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash128},
} {
_, err := createMp(s3client, bucket, "my-obj", withChecksum(test.algo), withChecksumType(test.chType))
if err := checkApiErr(err, s3err.GetChecksumSchemaMismatchErr(test.algo, test.chType)); err != nil {
@@ -414,6 +419,11 @@ func CreateMultipartUpload_valid_algo_type(s *S3Conf) error {
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha512},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmMd5},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash64},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash3},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash128},
// full object type
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc64nvme},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc32},
tests/integration/GetObject.go
+45 -131
View File
@@ -233,38 +233,13 @@ func GetObject_invalid_parent(s *S3Conf) error {
func GetObject_checksums(s *S3Conf) error {
testName := "GetObject_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -272,7 +247,7 @@ func GetObject_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -281,28 +256,11 @@ func GetObject_checksums(s *S3Conf) error {
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v",
el.key, types.ChecksumTypeFullObject, res.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v",
key, types.ChecksumTypeFullObject, res.ChecksumType)
}
if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32))
}
if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C))
}
if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1))
}
if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256))
}
if getString(res.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC64NVME), getString(res.ChecksumCRC64NVME))
if got, expected := getString(getGetObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got)
}
}
@@ -313,43 +271,31 @@ func GetObject_checksums(s *S3Conf) error {
func GetObject_dir_object_checksum(s *S3Conf) error {
testName := "GetObject_dir_object_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
for i, obj := range []struct {
key string
expectedSum string
for i, test := range []struct {
checksumAlgo types.ChecksumAlgorithm
expectedSum string
}{
{
key: "obj-1/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3/",
expectedSum: "AAAAAAAAAAA=",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
{
key: "obj-4/",
expectedSum: "2jmj7l5rSw0yVb/vlWAYkK/YBwk=",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-5/",
expectedSum: "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{types.ChecksumAlgorithmCrc32, "AAAAAA=="},
{types.ChecksumAlgorithmCrc32c, "AAAAAA=="},
{types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="},
{types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="},
{types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},
{types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="},
{types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="},
{types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="},
{types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="},
{types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, &s3.PutObjectInput{
key := fmt.Sprintf("obj-%v/", i+1)
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj.key,
ChecksumAlgorithm: obj.checksumAlgo,
})
Key: &key,
ChecksumAlgorithm: test.checksumAlgo,
}
setPutObjectChecksum(input, test.checksumAlgo, &test.expectedSum)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, input)
cancel()
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
@@ -358,7 +304,7 @@ func GetObject_dir_object_checksum(s *S3Conf) error {
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &obj.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -368,25 +314,13 @@ func GetObject_dir_object_checksum(s *S3Conf) error {
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("test %v failed: expected the %v object checksum type to be %v, instaed got %v",
i+1, obj.key, types.ChecksumTypeFullObject, res.ChecksumType)
i+1, key, types.ChecksumTypeFullObject, res.ChecksumType)
}
var gotSum *string
switch obj.checksumAlgo {
case types.ChecksumAlgorithmCrc32:
gotSum = res.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
gotSum = res.ChecksumCRC32C
case types.ChecksumAlgorithmCrc64nvme:
gotSum = res.ChecksumCRC64NVME
case types.ChecksumAlgorithmSha1:
gotSum = res.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
gotSum = res.ChecksumSHA256
}
gotSum := getGetObjectChecksum(res, test.checksumAlgo)
if getString(gotSum) != obj.expectedSum {
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, obj.checksumAlgo, obj.expectedSum, getString(gotSum))
if getString(gotSum) != test.expectedSum {
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, test.checksumAlgo, test.expectedSum, getString(gotSum))
}
}
@@ -885,20 +819,10 @@ func GetObject_not_enabled_checksum_mode(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getGetObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -1704,20 +1628,10 @@ func GetObject_ranged_with_checksum_mode(s *S3Conf) error {
testName := "GetObject_ranged_with_checksum_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
checkNoChecksums := func(res *s3.GetObjectOutput) error {
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getGetObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
}
tests/integration/GetObjectAttributes.go
+11 -53
View File
@@ -235,38 +235,13 @@ func GetObjectAttributes_existing_object(s *S3Conf) error {
func GetObjectAttributes_checksums(s *S3Conf) error {
testName := "GetObjectAttributes_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -274,7 +249,7 @@ func GetObjectAttributes_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesChecksum,
},
@@ -288,28 +263,11 @@ func GetObjectAttributes_checksums(s *S3Conf) error {
return fmt.Errorf("expected non-nil checksum in the response")
}
if res.Checksum.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v",
el.key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v",
key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType)
}
if getString(res.Checksum.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.Checksum.ChecksumCRC32))
}
if getString(res.Checksum.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32C), getString(res.Checksum.ChecksumCRC32C))
}
if getString(res.Checksum.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA1), getString(res.Checksum.ChecksumSHA1))
}
if getString(res.Checksum.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA256), getString(res.Checksum.ChecksumSHA256))
}
if getString(res.Checksum.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC64NVME), getString(res.Checksum.ChecksumCRC64NVME))
if got, expected := getString(getObjectAttributesChecksum(res.Checksum, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %s, instead got %s", checksumAlgo, expected, got)
}
}
return nil
tests/integration/HeadObject.go
+18 -72
View File
@@ -150,17 +150,10 @@ func HeadObject_not_enabled_checksum_mode(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -170,38 +163,13 @@ func HeadObject_not_enabled_checksum_mode(s *S3Conf) error {
func HeadObject_checksums(s *S3Conf) error {
testName := "HeadObject_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*200), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*200), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -209,7 +177,7 @@ func HeadObject_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -218,22 +186,10 @@ func HeadObject_checksums(s *S3Conf) error {
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v", key, types.ChecksumTypeFullObject, res.ChecksumType)
}
if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32))
}
if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C))
}
if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1))
}
if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256))
}
if getString(res.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v", getString(out.res.ChecksumCRC64NVME), getString(res.ChecksumCRC64NVME))
if got, expected := getString(getHeadObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got)
}
}
@@ -1311,20 +1267,10 @@ func HeadObject_ranged_with_checksum_mode(s *S3Conf) error {
testName := "HeadObject_ranged_with_checksum_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
checkNoChecksums := func(res *s3.HeadObjectOutput) error {
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
}
tests/integration/ListMultipartUploads.go
+26
View File
@@ -612,6 +612,32 @@ func ListMultipartUploads_with_checksums(s *S3Conf) error {
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
{
obj: "obj-6",
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
{
obj: "obj-7",
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
{
obj: "obj-8",
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
{
obj: "obj-9",
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
// obj-91 to preserve the lexicographical order
{
obj: "obj-91",
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
} {
key := el.obj
mp, err := createMp(s3client, bucket, key, withChecksum(el.algo), withChecksumType(el.t))
tests/integration/ListObjects.go
+1 -1
View File
@@ -349,7 +349,7 @@ func ListObjects_with_checksum(s *S3Conf) error {
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: el,
}, s3client)
}, s3client, withPutObjectChecksumAlgo(el))
if err != nil {
return err
}
tests/integration/ListObjectsV2.go
+1 -1
View File
@@ -578,7 +578,7 @@ func ListObjectsV2_with_checksum(s *S3Conf) error {
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: el,
}, s3client)
}, s3client, withPutObjectChecksumAlgo(el))
if err != nil {
return err
}
tests/integration/PostObject.go
+5
View File
@@ -1070,6 +1070,11 @@ func PostObject_checksums_success(s *S3Conf) error {
{"x-amz-checksum-crc64nvme", "rsrzr5yYqFU="},
{"x-amz-checksum-sha1", "9I3YU4IIYIFsddVND1hNyGMyenw="},
{"x-amz-checksum-sha256", "kW8AJ6V1B0znKjMXd8NHjWUT94alkb2JLaGld78jNfk="},
{"x-amz-checksum-sha512", "Dh4h7PEF7IU9JNcohnrXBhPCFmOkaTB0sqNhnBvTnWa1iMM3I7tGbHJCToDjymPCSQeKs0e6uUKFAOfuQwWdDQ=="},
{"x-amz-checksum-md5", "63M6AMDJ0zbmVpGjerVCkw=="},
{"x-amz-checksum-xxhash64", "+lb36/ER8bo="},
{"x-amz-checksum-xxhash3", "jw+pSh/pbMQ="},
{"x-amz-checksum-xxhash128", "8BLDqqIWji+ITOsp/JjN/Q=="},
} {
resp, err := sendPostObject(PostRequestConfig{
bucket: bucket,
tests/integration/PutObject.go
+58 -185
View File
@@ -702,98 +702,31 @@ func PutObject_invalid_checksum_header(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for i, el := range []struct {
algo string
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
// CRC32 tests
{
algo: "crc32",
crc32: getPtr(""),
},
{
algo: "crc32",
crc32: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32",
crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32
},
// CRC32C tests
{
algo: "crc32c",
crc32c: getPtr(""),
},
{
algo: "crc32c",
crc32c: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32c",
crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c
},
// SHA1 tests
{
algo: "sha1",
sha1: getPtr(""),
},
{
algo: "sha1",
sha1: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha1",
sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1
},
// SHA256 tests
{
algo: "sha256",
sha256: getPtr(""),
},
{
algo: "sha256",
sha256: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha256",
sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56
},
// CRC64Nvme tests
{
algo: "crc64nvme",
sha256: getPtr(""),
},
{
algo: "crc64nvme",
sha256: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc64nvme",
sha256: getPtr("ZHNhZmRzYWZzZGFmZHNhZg=="), // valid base64 but not crc64nvme
},
} {
_, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
}, s3client)
i := 0
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
// tests against:
// - empty string
// - invalid base64
// - valid base64, but invalid checksum
for _, checksum := range []string{"", "invalid_base64!", "c2RhZnNhZGZzZGFm"} {
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}
setPutObjectChecksum(input, algo, getPtr(checksum))
// FIXME: The error message for PutObject is not properly serialized by the sdk
// References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921
_, err := putObjectWithData(int64((i+1)*100), input, s3client)
i++
// if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil {
// return err
// }
if err := checkSdkApiErr(err, "InvalidRequest"); err != nil {
return err
// FIXME: The error message for PutObject is not properly serialized by the sdk
// References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921
// if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil {
// return err
// }
if err := checkSdkApiErr(err, "InvalidRequest"); err != nil {
return err
}
}
}
@@ -807,62 +740,33 @@ func PutObject_incorrect_checksums(s *S3Conf) error {
obj := "my-obj"
dirObj := "dir-object/"
for i, el := range []struct {
algo types.ChecksumAlgorithm
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
{
algo: types.ChecksumAlgorithmCrc32,
crc32: getPtr("DUoRhQ=="),
},
{
algo: types.ChecksumAlgorithmCrc32c,
crc32c: getPtr("yZRlqg=="),
},
{
algo: types.ChecksumAlgorithmSha1,
sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="),
},
{
algo: types.ChecksumAlgorithmSha256,
sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="),
},
{
algo: types.ChecksumAlgorithmCrc64nvme,
crc64nvme: getPtr("sV264W+gYBI="),
},
} {
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
wrongChecksum, err := wrongChecksumForAlgorithm(algo)
if err != nil {
return err
}
// test for file object
_, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
}, s3client)
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}
setPutObjectChecksum(input, algo, &wrongChecksum)
_, err = putObjectWithData(int64((i+1)*100), input, s3client)
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil {
return err
}
// test for directory object
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &bucket,
Key: &dirObj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
})
input = &s3.PutObjectInput{
Bucket: &bucket,
Key: &dirObj,
}
setPutObjectChecksum(input, algo, &wrongChecksum)
_, err = s3client.PutObject(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil {
return err
}
}
@@ -952,13 +856,20 @@ func PutObject_dir_object_checksums_success(s *S3Conf) error {
{types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="},
{types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="},
{types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},
{types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="},
{types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="},
{types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="},
{types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="},
{types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.PutObject(ctx, &s3.PutObjectInput{
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: getPtr(fmt.Sprintf("obj-%v/", i)),
ChecksumAlgorithm: test.algo,
})
}
setPutObjectChecksum(input, test.algo, &test.checksumValue)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.PutObject(ctx, input)
cancel()
if err != nil {
return err
@@ -968,27 +879,8 @@ func PutObject_dir_object_checksums_success(s *S3Conf) error {
return fmt.Errorf("expected the checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType)
}
switch test.algo {
case types.ChecksumAlgorithmCrc32:
if getString(res.ChecksumCRC32) != test.checksumValue {
return fmt.Errorf("expected the crc32 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32))
}
case types.ChecksumAlgorithmCrc32c:
if getString(res.ChecksumCRC32C) != test.checksumValue {
return fmt.Errorf("expected the crc32c checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32C))
}
case types.ChecksumAlgorithmSha1:
if getString(res.ChecksumSHA1) != test.checksumValue {
return fmt.Errorf("expected the sha1 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA1))
}
case types.ChecksumAlgorithmSha256:
if getString(res.ChecksumSHA256) != test.checksumValue {
return fmt.Errorf("expected the sha256 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA256))
}
case types.ChecksumAlgorithmCrc64nvme:
if getString(res.ChecksumCRC64NVME) != test.checksumValue {
return fmt.Errorf("expected the crc64nvme checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC64NVME))
}
if got := getString(getPutObjectChecksum(res, test.algo)); got != test.checksumValue {
return fmt.Errorf("expected the %s checksum value to be %s, instead got %s", test.algo, test.checksumValue, got)
}
}
@@ -1006,7 +898,7 @@ func PutObject_checksums_success(s *S3Conf) error {
Bucket: &bucket,
Key: &obj,
ChecksumAlgorithm: algo,
}, s3client)
}, s3client, withPutObjectChecksumAlgo(algo))
if err != nil {
return err
}
@@ -1015,27 +907,8 @@ func PutObject_checksums_success(s *S3Conf) error {
return fmt.Errorf("expected the object checksum type to be %v, instead got %v", types.ChecksumTypeFullObject, res.res.ChecksumType)
}
switch algo {
case types.ChecksumAlgorithmCrc32:
if res.res.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum in the response")
}
case types.ChecksumAlgorithmCrc32c:
if res.res.ChecksumCRC32C == nil {
return fmt.Errorf("expected non empty crc32c checksum in the response")
}
case types.ChecksumAlgorithmSha1:
if res.res.ChecksumSHA1 == nil {
return fmt.Errorf("expected non empty sha1 checksum in the response")
}
case types.ChecksumAlgorithmSha256:
if res.res.ChecksumSHA256 == nil {
return fmt.Errorf("expected non empty sha256 checksum in the response")
}
case types.ChecksumAlgorithmCrc64nvme:
if res.res.ChecksumCRC64NVME == nil {
return fmt.Errorf("expected non empty crc64nvme checksum in the response")
}
if getPutObjectChecksum(res.res, algo) == nil {
return fmt.Errorf("expected non empty %s checksum in the response", algo)
}
}
tests/integration/UploadPart.go
+51 -213
View File
@@ -155,95 +155,26 @@ func UploadPart_invalid_checksum_header(s *S3Conf) error {
partNumber := int32(1)
for _, el := range []struct {
algo string
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
// CRC32 tests
{
algo: "crc32",
crc32: getPtr(""),
},
{
algo: "crc32",
crc32: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32",
crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32
},
// CRC32C tests
{
algo: "crc32c",
crc32c: getPtr(""),
},
{
algo: "crc32c",
crc32c: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32c",
crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c
},
// SHA1 tests
{
algo: "sha1",
sha1: getPtr(""),
},
{
algo: "sha1",
sha1: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha1",
sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1
},
// SHA256 tests
{
algo: "sha256",
sha256: getPtr(""),
},
{
algo: "sha256",
sha256: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha256",
sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56
},
// CRC64NVME tests
{
algo: "crc64nvme",
crc64nvme: getPtr(""),
},
{
algo: "crc64nvme",
crc64nvme: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc64nvme",
crc64nvme: getPtr("ZHNhZmRzYWZzZGFmZHNhZg=="), // valid base64 but not crc64nvme
},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
PartNumber: &partNumber,
UploadId: mp.UploadId,
})
cancel()
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", el.algo))); err != nil {
return err
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
// tests against:
// - empty string
// - invalid base64
// - valid base64, but invalid checksum
for _, checksum := range []string{"", "invalid_base64!", "c2RhZnNhZGZzZGFm"} {
input := &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
PartNumber: &partNumber,
UploadId: mp.UploadId,
}
setUploadPartChecksum(input, algo, getPtr(checksum))
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.UploadPart(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(checksumHeaderName(algo))); err != nil {
return err
}
}
}
@@ -336,36 +267,13 @@ func UploadPart_incorrect_checksums(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for _, el := range []struct {
algo types.ChecksumAlgorithm
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
{
algo: types.ChecksumAlgorithmCrc32,
crc32: getPtr("DUoRhQ=="),
},
{
algo: types.ChecksumAlgorithmCrc32c,
crc32c: getPtr("yZRlqg=="),
},
{
algo: types.ChecksumAlgorithmSha1,
sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="),
},
{
algo: types.ChecksumAlgorithmSha256,
sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="),
},
{
algo: types.ChecksumAlgorithmCrc64nvme,
crc64nvme: getPtr("MN2ofvMjpIQ="),
},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo))
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
wrongChecksum, err := wrongChecksumForAlgorithm(algo)
if err != nil {
return err
}
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
@@ -374,20 +282,17 @@ func UploadPart_incorrect_checksums(s *S3Conf) error {
partNumber := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
UploadId: mp.UploadId,
PartNumber: &partNumber,
Body: body,
})
input := &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
Body: body,
}
setUploadPartChecksum(input, algo, &wrongChecksum)
_, err = s3client.UploadPart(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil {
return err
}
}
@@ -479,6 +384,11 @@ func UploadPart_no_checksum_with_composite_checksum_type(s *S3Conf) error {
types.ChecksumAlgorithmCrc32c,
types.ChecksumAlgorithmSha1,
types.ChecksumAlgorithmSha256,
types.ChecksumAlgorithmSha512,
types.ChecksumAlgorithmMd5,
types.ChecksumAlgorithmXxhash64,
types.ChecksumAlgorithmXxhash3,
types.ChecksumAlgorithmXxhash128,
} {
mp, err := createMp(customClient, bucket, obj, withChecksum(algo), withChecksumType(types.ChecksumTypeComposite))
if err != nil {
@@ -493,8 +403,8 @@ func UploadPart_no_checksum_with_composite_checksum_type(s *S3Conf) error {
})
}
func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Conf) error {
testName := "UploadPart_should_calculate_checksum_if_only_algorithm_is_provided"
func UploadPart_with_checksums_success(s *S3Conf) error {
testName := "UploadPart_with_checksums_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
customClient := s3.NewFromConfig(s.Config(), func(o *s3.Options) {
o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset
@@ -512,6 +422,11 @@ func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Con
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha512},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmMd5},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash64},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash3},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash128},
} {
mp, err := createMp(customClient, bucket, obj, withChecksum(test.chAlgo), withChecksumType(test.chType))
if err != nil {
@@ -527,91 +442,14 @@ func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Con
return fmt.Errorf("expected 1 uploaded part, instaed got %d", len(parts))
}
part := parts[0]
switch test.chAlgo {
case types.ChecksumAlgorithmCrc32:
if getString(part.ChecksumCRC32) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32))
}
case types.ChecksumAlgorithmCrc32c:
if getString(part.ChecksumCRC32C) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32C))
}
case types.ChecksumAlgorithmCrc64nvme:
if getString(part.ChecksumCRC64NVME) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC64NVME))
}
case types.ChecksumAlgorithmSha1:
if getString(part.ChecksumSHA1) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA1))
}
case types.ChecksumAlgorithmSha256:
if getString(part.ChecksumSHA256) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA256))
}
if got := getString(getPartChecksum(parts[0], test.chAlgo)); got != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, got)
}
}
return nil
})
}
func UploadPart_with_checksums_success(s *S3Conf) error {
testName := "UploadPart_with_checksums_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
partNumber := int32(1)
data := make([]byte, i*100)
rand.Read(data)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
ChecksumAlgorithm: algo,
UploadId: mp.UploadId,
PartNumber: &partNumber,
Body: bytes.NewReader(data),
})
cancel()
if err != nil {
return err
}
switch algo {
case types.ChecksumAlgorithmCrc32:
if res.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum in the response")
}
case types.ChecksumAlgorithmCrc32c:
if res.ChecksumCRC32C == nil {
return fmt.Errorf("expected non empty crc32c checksum in the response")
}
case types.ChecksumAlgorithmSha1:
if res.ChecksumSHA1 == nil {
return fmt.Errorf("expected non empty sha1 checksum in the response")
}
case types.ChecksumAlgorithmSha256:
if res.ChecksumSHA256 == nil {
return fmt.Errorf("expected non empty sha256 checksum in the response")
}
case types.ChecksumAlgorithmCrc64nvme:
if res.ChecksumCRC64NVME == nil {
return fmt.Errorf("expected non empty crc64nvme checksum in the response")
}
}
}
return nil
})
}
func UploadPart_non_existing_key(s *S3Conf) error {
testName := "UploadPart_non_existing_key"
partNumber := int32(1)
tests/integration/UploadPartCopy.go
+83 -110
View File
@@ -845,57 +845,50 @@ func UploadPartCopy_conditional_reads(s *S3Conf) error {
func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error {
testName := "UploadPartCopy_should_copy_the_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
srcObj := "source-object"
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
obj := fmt.Sprintf("my-obj-%v", i+1)
srcObj := fmt.Sprintf("source-object-%v", i+1)
mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32))
if err != nil {
return err
}
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
out, err := putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: types.ChecksumAlgorithmCrc32,
}, s3client)
if err != nil {
return err
}
out, err := putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: algo,
}, s3client, withPutObjectChecksumAlgo(algo))
if err != nil {
return err
}
partNumber := int32(1)
partNumber := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
if getString(res.CopyPartResult.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.CopyPartResult.ChecksumCRC32))
}
if res.CopyPartResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32C)
}
if res.CopyPartResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA1)
}
if res.CopyPartResult.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA256)
}
if res.CopyPartResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC64NVME)
if got, expected := getString(getUploadPartCopyChecksum(res.CopyPartResult, algo)), getString(getPutObjectChecksum(out.res, algo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", algo, expected, got)
}
for _, otherAlgo := range types.ChecksumAlgorithmCrc32.Values() {
if otherAlgo == algo {
continue
}
if checksum := getUploadPartCopyChecksum(res.CopyPartResult, otherAlgo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", otherAlgo, *checksum)
}
}
}
return nil
@@ -937,25 +930,10 @@ func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error {
return err
}
if res.CopyPartResult.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32)
}
if res.CopyPartResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32C)
}
if res.CopyPartResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA1)
}
if res.CopyPartResult.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA256)
}
if res.CopyPartResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getUploadPartCopyChecksum(res.CopyPartResult, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -965,56 +943,51 @@ func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error {
func UploadPartCopy_should_calculate_the_checksum(s *S3Conf) error {
testName := "UploadPartCopy_should_calculate_the_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
srcObj := "source-object"
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
obj := fmt.Sprintf("my-obj-%v", i+1)
srcObj := fmt.Sprintf("source-object-%v", i+1)
mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha256))
if err != nil {
return err
}
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
_, err = putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: types.ChecksumAlgorithmSha1, // different from the mp checksum (sha256)
}, s3client)
if err != nil {
return err
}
_, err = putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: types.ChecksumAlgorithmSha1,
}, s3client)
if err != nil {
return err
}
partNumber := int32(1)
partNumber := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
if res.CopyPartResult.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32)
}
if res.CopyPartResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32C)
}
if res.CopyPartResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC64NVME)
}
if res.CopyPartResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA1)
}
if getString(res.CopyPartResult.ChecksumSHA256) == "" {
return fmt.Errorf("expected non empty sha256 checksum")
for _, otherAlgo := range types.ChecksumAlgorithmCrc32.Values() {
checksum := getUploadPartCopyChecksum(res.CopyPartResult, otherAlgo)
if otherAlgo == algo {
if getString(checksum) == "" {
return fmt.Errorf("expected non empty %s checksum", algo)
}
continue
}
if checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", otherAlgo, *checksum)
}
}
}
return nil
tests/integration/group-tests.go
-2
View File
@@ -445,7 +445,6 @@ func TestUploadPart(ts *TestState) {
ts.Run(UploadPart_incorrect_checksums)
ts.Run(UploadPart_no_checksum_with_full_object_checksum_type)
ts.Run(UploadPart_no_checksum_with_composite_checksum_type)
ts.Run(UploadPart_should_calculate_checksum_if_only_algorithm_is_provided)
ts.Run(UploadPart_with_checksums_success)
}
ts.Run(UploadPart_success)
@@ -1589,7 +1588,6 @@ func GetIntTests() IntTests {
"UploadPart_incorrect_checksums": UploadPart_incorrect_checksums,
"UploadPart_no_checksum_with_full_object_checksum_type": UploadPart_no_checksum_with_full_object_checksum_type,
"UploadPart_no_checksum_with_composite_checksum_type": UploadPart_no_checksum_with_composite_checksum_type,
"UploadPart_should_calculate_checksum_if_only_algorithm_is_provided": UploadPart_should_calculate_checksum_if_only_algorithm_is_provided,
"UploadPart_with_checksums_success": UploadPart_with_checksums_success,
"UploadPart_success": UploadPart_success,
"UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket,
tests/integration/signed_streaming_payload_trailer.go
+27 -23
View File
@@ -108,24 +108,17 @@ func SignedStreamingPayloadTrailer_invalid_checksum(s *S3Conf) error {
testName := "SignedStreamingPayloadTrailer_invalid_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
trailerHdr string
trailingChecksum string
}{
{"x-amz-checksum-crc32", "x-amz-checksum-crc32:invalid"},
{"x-amz-checksum-crc32c", "x-amz-checksum-crc32c:invalid"},
{"x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:invalid"},
{"x-amz-checksum-sha1", "x-amz-checksum-sha1:invalid"},
{"x-amz-checksum-sha256", "x-amz-checksum-sha256:invalid"},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": test.trailerHdr,
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
trailerHdr := checksumHeaderName(algo)
trailingChecksum := fmt.Sprintf("%s:invalid", trailerHdr)
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": trailerHdr,
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(test.trailerHdr), apiErr); err != nil {
if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(trailerHdr), apiErr); err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
}
@@ -139,18 +132,24 @@ func SignedStreamingPayloadTrailer_bad_digest(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
algo types.ChecksumAlgorithm
trailerHdr string
trailingChecksum string
algo types.ChecksumAlgorithm
checksum string
}{
{types.ChecksumAlgorithmCrc32, "x-amz-checksum-crc32", "x-amz-checksum-crc32:NhCmhg=="},
{types.ChecksumAlgorithmCrc32c, "x-amz-checksum-crc32c", "x-amz-checksum-crc32c:+Cy97w=="},
{types.ChecksumAlgorithmCrc64nvme, "x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:QFRKMGE3tuw="},
{types.ChecksumAlgorithmSha1, "x-amz-checksum-sha1", "x-amz-checksum-sha1:qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{types.ChecksumAlgorithmSha256, "x-amz-checksum-sha256", "x-amz-checksum-sha256:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
{types.ChecksumAlgorithmCrc32, "NhCmhg=="},
{types.ChecksumAlgorithmCrc32c, "+Cy97w=="},
{types.ChecksumAlgorithmCrc64nvme, "QFRKMGE3tuw="},
{types.ChecksumAlgorithmSha1, "qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{types.ChecksumAlgorithmSha256, "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
{types.ChecksumAlgorithmSha512, "mD1D3f9tqQ9qXTthckRqH/4ii4A/5k/dXc+rVkYHioloUf6C9iPJ1uVlSz0vNjoE7BfPtitgdDepx8Ey1RHlIg=="},
{types.ChecksumAlgorithmMd5, "Q3uTDbhLgHnC3YBKcZNrXw=="},
{types.ChecksumAlgorithmXxhash64, "dVz0EuroMyA="},
{types.ChecksumAlgorithmXxhash3, "OHmyyhTeg1E="},
{types.ChecksumAlgorithmXxhash128, "6fhA2KZsN830zx0e5P183w=="},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": test.trailerHdr,
trailerHdr := checksumHeaderName(test.algo)
trailingChecksum := fmt.Sprintf("%s:%s", trailerHdr, test.checksum)
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": trailerHdr,
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
@@ -178,6 +177,11 @@ func SignedStreamingPayloadTrailer_success(s *S3Conf) error {
{"x-amz-checksum-crc64nvme", "dYnI3/Fh0gM="},
{"x-amz-checksum-sha1", "8O8FwCfmd5fCbCBvH09mrKMVoHU="},
{"x-amz-checksum-sha256", "OoSow5X4zTIPl27MtdFdYT+9O3C367C75+Cb2MFtRBc="},
{"x-amz-checksum-sha512", "uiz+VuENyLjzFxWrdmbN/NdIdldj/V3saJF6FsckcYu6xu26fM0CfAaTySaoFJmIdI2m5wbYMJxtShQ1PXk3Tg=="},
{"x-amz-checksum-md5", "EtWtthXtCk3RxUiXKw+ydw=="},
{"x-amz-checksum-xxhash64", "McmxUfNFLUs="},
{"x-amz-checksum-xxhash3", "tLh+aNltTM4="},
{"x-amz-checksum-xxhash128", "MXqrLVobrQkTBf82+7i9AQ=="},
} {
headers, apiErr, err := testSignedStreamingObjectPut(
s,
tests/integration/unsigned_streaming_payload_trailer.go
+30 -9
View File
@@ -134,6 +134,11 @@ func UnsignedStreamingPayloadTrailer_multiple_checksum_headers(s *S3Conf) error
{"crc64nvme", "QFRKMGE3tuw="},
{"sha1", "qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{"sha256", "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
{"sha512", "uiz+VuENyLjzFxWrdmbN/NdIdldj/V3saJF6FsckcYu6xu26fM0CfAaTySaoFJmIdI2m5wbYMJxtShQ1PXk3Tg=="},
{"md5", "EtWtthXtCk3RxUiXKw+ydw=="},
{"xxhash64", "McmxUfNFLUs="},
{"xxhash3", "tLh+aNltTM4="},
{"xxhash128", "MXqrLVobrQkTBf82+7i9AQ=="},
} {
reqHeaders := map[string]string{
"x-amz-decoded-content-length": "5",
@@ -348,6 +353,11 @@ func UnsignedStreamingPayloadTrailer_no_payload_trailer_only_headers(s *S3Conf)
{"crc64nvme", "SmzZ/LTp1CA="},
{"sha1", "L7XhNBn8iSRoZeejJPR27GJOh0A="},
{"sha256", "fRpUEnsiJQL1t5tfsIAwYRUqRPkrN+I8ZSe69mXU2po="},
{"sha512", "1xakGIVptoqxtt+sF45XARTN8Oo6HMDjFIbD5BJBvGp2Qk6MN6sm8Jb8he+YhsjLY0GH9P3f9kX7CZ8f9UxrjA=="},
{"md5", "esZsDxSN6VGbi9JkMSxNZA=="},
{"xxhash64", "GGCUDikCgi0="},
{"xxhash3", "WkDcP9RMBS8="},
{"xxhash128", "Kq/YOGmlnDE/55jA7aptxg=="},
} {
csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key)
reqHeaders := map[string]string{
@@ -387,6 +397,11 @@ func UnsignedStreamingPayloadTrailer_success_both_sdk_algo_and_trailer(s *S3Conf
{"crc64nvme", "SmzZ/LTp1CA="},
{"sha1", "L7XhNBn8iSRoZeejJPR27GJOh0A="},
{"sha256", "fRpUEnsiJQL1t5tfsIAwYRUqRPkrN+I8ZSe69mXU2po="},
{"sha512", "1xakGIVptoqxtt+sF45XARTN8Oo6HMDjFIbD5BJBvGp2Qk6MN6sm8Jb8he+YhsjLY0GH9P3f9kX7CZ8f9UxrjA=="},
{"md5", "esZsDxSN6VGbi9JkMSxNZA=="},
{"xxhash64", "GGCUDikCgi0="},
{"xxhash3", "WkDcP9RMBS8="},
{"xxhash128", "Kq/YOGmlnDE/55jA7aptxg=="},
} {
csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key)
reqHeaders := map[string]string{
@@ -508,23 +523,29 @@ func UnsignedStreamingPayloadTrailer_UploadPart_success_with_trailer(s *S3Conf)
object := "my-object"
for i, test := range []struct {
key string
algo types.ChecksumAlgorithm
value string
}{
{"crc32", "QWaN2w=="},
{"crc32c", "R/I7iQ=="},
{"crc64nvme", "dPVWc2vU1+Q="},
{"sha1", "YR/1TvTYOJz5gtqVFoBJBtmTibY="},
{"sha256", "eXuwq/95jXIAr3aF3KeQHt/8Ur8mUA1b2XKCZY7iQVI="},
{types.ChecksumAlgorithmCrc32, "QWaN2w=="},
{types.ChecksumAlgorithmCrc32c, "R/I7iQ=="},
{types.ChecksumAlgorithmCrc64nvme, "dPVWc2vU1+Q="},
{types.ChecksumAlgorithmSha1, "YR/1TvTYOJz5gtqVFoBJBtmTibY="},
{types.ChecksumAlgorithmSha256, "eXuwq/95jXIAr3aF3KeQHt/8Ur8mUA1b2XKCZY7iQVI="},
{types.ChecksumAlgorithmSha512, "6tQCd6X50h2wXt5deAY0eKF5Xb1LLbSvkZt+Bqczz8bd1+rH+VYZSgWhjOG4zJ41K0kLmDQxuDyOeHMKqKQGNA=="},
{types.ChecksumAlgorithmMd5, "Mb+5cwrlGvc5U7pyDIZg1w=="},
{types.ChecksumAlgorithmXxhash64, "vYOmFpsGXtY="},
{types.ChecksumAlgorithmXxhash3, "9J6xx21X+f8="},
{types.ChecksumAlgorithmXxhash128, "menoCZOCZ2Acv4XMxaUR9w=="},
} {
mp, err := createMp(s3client, bucket, object, withChecksum(types.ChecksumAlgorithm(strings.ToUpper(test.key))))
mp, err := createMp(s3client, bucket, object, withChecksum(test.algo))
if err != nil {
return err
}
csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key)
csumHdr := checksumHeaderName(test.algo)
reqHeaders := map[string]string{
"x-amz-decoded-content-length": "10",
"x-amz-sdk-checksum-algorithm": test.key,
"x-amz-sdk-checksum-algorithm": strings.ToLower(string(test.algo)),
"x-amz-trailer": csumHdr,
}
body := bytes.NewBuffer([]byte("A\r\ndummy data\r\n0\r\n"))
tests/integration/utils.go
+449 -27
View File
@@ -22,6 +22,7 @@ import (
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
@@ -53,7 +54,9 @@ import (
"github.com/aws/smithy-go"
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
)
@@ -547,21 +550,46 @@ func hasPrefixName(prefixes []types.CommonPrefix, names []string) bool {
return true
}
type putObjectCfg struct {
checksumAlgorithm types.ChecksumAlgorithm
}
type putObjectOpt func(*putObjectCfg)
func withPutObjectChecksumAlgo(algo types.ChecksumAlgorithm) putObjectOpt {
return func(poc *putObjectCfg) { poc.checksumAlgorithm = algo }
}
type putObjectOutput struct {
csum [32]byte
data []byte
res *s3.PutObjectOutput
}
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) {
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client, opts ...putObjectOpt) (*putObjectOutput, error) {
cfg := &putObjectCfg{}
for _, opt := range opts {
opt(cfg)
}
var csum [32]byte
var data []byte
if input.Body == nil && lgth != 0 {
data = make([]byte, lgth)
rand.Read(data)
csum = sha256.Sum256(data)
r := bytes.NewReader(data)
input.Body = r
if cfg.checksumAlgorithm != "" {
hasher, err := NewHasher(cfg.checksumAlgorithm)
if err != nil {
return nil, err
}
hasher.Write(data)
sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
setPutObjectChecksum(input, cfg.checksumAlgorithm, &sum)
}
input.Body = bytes.NewReader(data)
}
ctx, cancel := context.WithTimeout(context.Background(), longTimeout)
@@ -713,6 +741,36 @@ func compareParts(parts1, parts2 []types.Part) bool {
return false
}
}
if prt.ChecksumSHA512 != nil {
if *prt.ChecksumSHA512 != getString(parts2[i].ChecksumSHA512) {
fmt.Printf("sha512 checksums are not equal, %v != %v\n", *prt.ChecksumSHA512, getString(parts2[i].ChecksumSHA512))
return false
}
}
if prt.ChecksumMD5 != nil {
if *prt.ChecksumMD5 != getString(parts2[i].ChecksumMD5) {
fmt.Printf("md5 checksums are not equal, %v != %v\n", *prt.ChecksumMD5, getString(parts2[i].ChecksumMD5))
return false
}
}
if prt.ChecksumXXHASH64 != nil {
if *prt.ChecksumXXHASH64 != getString(parts2[i].ChecksumXXHASH64) {
fmt.Printf("xxhash64 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH64, getString(parts2[i].ChecksumXXHASH64))
return false
}
}
if prt.ChecksumXXHASH3 != nil {
if *prt.ChecksumXXHASH3 != getString(parts2[i].ChecksumXXHASH3) {
fmt.Printf("xxhash3 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH3, getString(parts2[i].ChecksumXXHASH3))
return false
}
}
if prt.ChecksumXXHASH128 != nil {
if *prt.ChecksumXXHASH128 != getString(parts2[i].ChecksumXXHASH128) {
fmt.Printf("xxhash128 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH128, getString(parts2[i].ChecksumXXHASH128))
return false
}
}
}
return true
}
@@ -775,6 +833,299 @@ func getPtr[T any](str T) *T {
return &str
}
func checksumHeaderName(algo types.ChecksumAlgorithm) string {
return fmt.Sprintf("x-amz-checksum-%s", strings.ToLower(string(algo)))
}
type checksumFields struct {
CRC32 **string
CRC32C **string
SHA1 **string
SHA256 **string
CRC64NVME **string
SHA512 **string
MD5 **string
XXHASH64 **string
XXHASH3 **string
XXHASH128 **string
}
func selectChecksum(algo types.ChecksumAlgorithm, fields checksumFields) **string {
switch algo {
case types.ChecksumAlgorithmCrc32:
return fields.CRC32
case types.ChecksumAlgorithmCrc32c:
return fields.CRC32C
case types.ChecksumAlgorithmSha1:
return fields.SHA1
case types.ChecksumAlgorithmSha256:
return fields.SHA256
case types.ChecksumAlgorithmCrc64nvme:
return fields.CRC64NVME
case types.ChecksumAlgorithmSha512:
return fields.SHA512
case types.ChecksumAlgorithmMd5:
return fields.MD5
case types.ChecksumAlgorithmXxhash64:
return fields.XXHASH64
case types.ChecksumAlgorithmXxhash3:
return fields.XXHASH3
case types.ChecksumAlgorithmXxhash128:
return fields.XXHASH128
default:
return nil
}
}
func getChecksum(algo types.ChecksumAlgorithm, fields checksumFields) *string {
if checksum := selectChecksum(algo, fields); checksum != nil {
return *checksum
}
return nil
}
func setChecksum(algo types.ChecksumAlgorithm, fields checksumFields, checksum *string) {
if selected := selectChecksum(algo, fields); selected != nil {
*selected = checksum
}
}
func getPartChecksum(part types.Part, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &part.ChecksumCRC32,
CRC32C: &part.ChecksumCRC32C,
SHA1: &part.ChecksumSHA1,
SHA256: &part.ChecksumSHA256,
CRC64NVME: &part.ChecksumCRC64NVME,
SHA512: &part.ChecksumSHA512,
MD5: &part.ChecksumMD5,
XXHASH64: &part.ChecksumXXHASH64,
XXHASH3: &part.ChecksumXXHASH3,
XXHASH128: &part.ChecksumXXHASH128,
})
}
func setPartChecksum(part *types.Part, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &part.ChecksumCRC32,
CRC32C: &part.ChecksumCRC32C,
SHA1: &part.ChecksumSHA1,
SHA256: &part.ChecksumSHA256,
CRC64NVME: &part.ChecksumCRC64NVME,
SHA512: &part.ChecksumSHA512,
MD5: &part.ChecksumMD5,
XXHASH64: &part.ChecksumXXHASH64,
XXHASH3: &part.ChecksumXXHASH3,
XXHASH128: &part.ChecksumXXHASH128,
}, checksum)
}
func getCompletedPartChecksum(part types.CompletedPart, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &part.ChecksumCRC32,
CRC32C: &part.ChecksumCRC32C,
SHA1: &part.ChecksumSHA1,
SHA256: &part.ChecksumSHA256,
CRC64NVME: &part.ChecksumCRC64NVME,
SHA512: &part.ChecksumSHA512,
MD5: &part.ChecksumMD5,
XXHASH64: &part.ChecksumXXHASH64,
XXHASH3: &part.ChecksumXXHASH3,
XXHASH128: &part.ChecksumXXHASH128,
})
}
func completedPartFromPart(part types.Part) types.CompletedPart {
return types.CompletedPart{
ETag: part.ETag,
PartNumber: part.PartNumber,
ChecksumCRC32: part.ChecksumCRC32,
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumMD5: part.ChecksumMD5,
ChecksumXXHASH64: part.ChecksumXXHASH64,
ChecksumXXHASH3: part.ChecksumXXHASH3,
ChecksumXXHASH128: part.ChecksumXXHASH128,
}
}
func getPutObjectChecksum(out *s3.PutObjectOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func setPutObjectChecksum(in *s3.PutObjectInput, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &in.ChecksumCRC32,
CRC32C: &in.ChecksumCRC32C,
SHA1: &in.ChecksumSHA1,
SHA256: &in.ChecksumSHA256,
CRC64NVME: &in.ChecksumCRC64NVME,
SHA512: &in.ChecksumSHA512,
MD5: &in.ChecksumMD5,
XXHASH64: &in.ChecksumXXHASH64,
XXHASH3: &in.ChecksumXXHASH3,
XXHASH128: &in.ChecksumXXHASH128,
}, checksum)
}
func getGetObjectChecksum(out *s3.GetObjectOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func getHeadObjectChecksum(out *s3.HeadObjectOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func getObjectAttributesChecksum(out *types.Checksum, algo types.ChecksumAlgorithm) *string {
if out == nil {
return nil
}
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func getUploadPartChecksum(out *s3.UploadPartOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func setUploadPartChecksum(in *s3.UploadPartInput, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &in.ChecksumCRC32,
CRC32C: &in.ChecksumCRC32C,
SHA1: &in.ChecksumSHA1,
SHA256: &in.ChecksumSHA256,
CRC64NVME: &in.ChecksumCRC64NVME,
SHA512: &in.ChecksumSHA512,
MD5: &in.ChecksumMD5,
XXHASH64: &in.ChecksumXXHASH64,
XXHASH3: &in.ChecksumXXHASH3,
XXHASH128: &in.ChecksumXXHASH128,
}, checksum)
}
func getCompleteMultipartUploadChecksum(out *s3.CompleteMultipartUploadOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func setCompleteMultipartUploadChecksum(in *s3.CompleteMultipartUploadInput, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &in.ChecksumCRC32,
CRC32C: &in.ChecksumCRC32C,
SHA1: &in.ChecksumSHA1,
SHA256: &in.ChecksumSHA256,
CRC64NVME: &in.ChecksumCRC64NVME,
SHA512: &in.ChecksumSHA512,
MD5: &in.ChecksumMD5,
XXHASH64: &in.ChecksumXXHASH64,
XXHASH3: &in.ChecksumXXHASH3,
XXHASH128: &in.ChecksumXXHASH128,
}, checksum)
}
func getCopyObjectChecksum(result *types.CopyObjectResult, algo types.ChecksumAlgorithm) *string {
if result == nil {
return nil
}
return getChecksum(algo, checksumFields{
CRC32: &result.ChecksumCRC32,
CRC32C: &result.ChecksumCRC32C,
SHA1: &result.ChecksumSHA1,
SHA256: &result.ChecksumSHA256,
CRC64NVME: &result.ChecksumCRC64NVME,
SHA512: &result.ChecksumSHA512,
MD5: &result.ChecksumMD5,
XXHASH64: &result.ChecksumXXHASH64,
XXHASH3: &result.ChecksumXXHASH3,
XXHASH128: &result.ChecksumXXHASH128,
})
}
func getUploadPartCopyChecksum(result *types.CopyPartResult, algo types.ChecksumAlgorithm) *string {
if result == nil {
return nil
}
return getChecksum(algo, checksumFields{
CRC32: &result.ChecksumCRC32,
CRC32C: &result.ChecksumCRC32C,
SHA1: &result.ChecksumSHA1,
SHA256: &result.ChecksumSHA256,
CRC64NVME: &result.ChecksumCRC64NVME,
SHA512: &result.ChecksumSHA512,
MD5: &result.ChecksumMD5,
XXHASH64: &result.ChecksumXXHASH64,
XXHASH3: &result.ChecksumXXHASH3,
XXHASH128: &result.ChecksumXXHASH128,
})
}
// mp1 needs to be the response from the server
// mp2 needs to be the expected values
// The keys from the server are always converted to lowercase
@@ -921,7 +1272,8 @@ func compareDelObjects(list1, list2 []types.DeletedObject) bool {
func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) {
partSize := size / partCount
var hash hash.Hash
var objHasher hash.Hash
var partHasher hash.Hash
cfg := new(mpCfg)
for _, opt := range opts {
@@ -930,17 +1282,34 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
hash = crc32.NewIEEE()
objHasher = crc32.NewIEEE()
case types.ChecksumAlgorithmCrc32c:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
objHasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case types.ChecksumAlgorithmMd5:
objHasher = md5.New()
case types.ChecksumAlgorithmSha1:
hash = sha1.New()
objHasher = sha1.New()
case types.ChecksumAlgorithmSha256:
hash = sha256.New()
objHasher = sha256.New()
case types.ChecksumAlgorithmSha512:
objHasher = sha512.New()
case types.ChecksumAlgorithmCrc64nvme:
hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
objHasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
case types.ChecksumAlgorithmXxhash64:
objHasher = xxhash.New()
case types.ChecksumAlgorithmXxhash3:
objHasher = xxh3.New()
case types.ChecksumAlgorithmXxhash128:
objHasher = xxh3.New128()
default:
hash = sha256.New()
objHasher = sha256.New()
}
if cfg.checksumAlgorithm != "" {
partHasher, err = NewHasher(cfg.checksumAlgorithm)
if err != nil {
return nil, "", err
}
}
for partNumber := int64(1); partNumber <= partCount; partNumber++ {
@@ -952,18 +1321,28 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
partBuffer := make([]byte, partEnd-partStart+1)
rand.Read(partBuffer)
hash.Write(partBuffer)
objHasher.Write(partBuffer)
if partHasher != nil {
partHasher.Write(partBuffer)
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
pn := int32(partNumber)
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
input := &s3.UploadPartInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: &pn,
ChecksumAlgorithm: cfg.checksumAlgorithm,
})
}
if partHasher != nil {
partChecksum := base64.StdEncoding.EncodeToString(partHasher.Sum(nil))
setUploadPartChecksum(input, cfg.checksumAlgorithm, &partChecksum)
partHasher.Reset()
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := client.UploadPart(ctx, input)
cancel()
if err != nil {
return parts, "", err
@@ -975,22 +1354,11 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
Size: &partSize,
}
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
part.ChecksumCRC32 = out.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
part.ChecksumCRC32C = out.ChecksumCRC32C
case types.ChecksumAlgorithmSha1:
part.ChecksumSHA1 = out.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
part.ChecksumSHA256 = out.ChecksumSHA256
case types.ChecksumAlgorithmCrc64nvme:
part.ChecksumCRC64NVME = out.ChecksumCRC64NVME
}
setPartChecksum(&part, cfg.checksumAlgorithm, getUploadPartChecksum(out, cfg.checksumAlgorithm))
parts = append(parts, part)
}
sum := hash.Sum(nil)
sum := objHasher.Sum(nil)
if cfg.checksumAlgorithm == "" {
csum = hex.EncodeToString(sum[:])
@@ -1277,6 +1645,26 @@ func createObjVersions(client *s3.Client, bucket, object string, count int, opts
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmSha256,
}
case r.res.ChecksumSHA512 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmSha512,
}
case r.res.ChecksumMD5 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmMd5,
}
case r.res.ChecksumXXHASH64 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmXxhash64,
}
case r.res.ChecksumXXHASH3 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmXxhash3,
}
case r.res.ChecksumXXHASH128 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmXxhash128,
}
}
versions = append(versions, version)
@@ -2012,14 +2400,26 @@ func lockObject(client *s3.Client, mode objectLockMode, bucket, object, versionI
func NewHasher(algo types.ChecksumAlgorithm) (hash.Hash, error) {
var hasher hash.Hash
switch algo {
case types.ChecksumAlgorithmMd5:
hasher = md5.New()
case types.ChecksumAlgorithmSha256:
hasher = sha256.New()
case types.ChecksumAlgorithmSha512:
hasher = sha512.New()
case types.ChecksumAlgorithmSha1:
hasher = sha1.New()
case types.ChecksumAlgorithmCrc32:
hasher = crc32.NewIEEE()
case types.ChecksumAlgorithmCrc32c:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case types.ChecksumAlgorithmCrc64nvme:
hasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
case types.ChecksumAlgorithmXxhash64:
hasher = xxhash.New()
case types.ChecksumAlgorithmXxhash3:
hasher = xxh3.New()
case types.ChecksumAlgorithmXxhash128:
hasher = xxh3.New128()
default:
return nil, fmt.Errorf("unsupported hash algorithm: %s", algo)
}
@@ -2027,6 +2427,28 @@ func NewHasher(algo types.ChecksumAlgorithm) (hash.Hash, error) {
return hasher, nil
}
func wrongChecksumForAlgorithm(algo types.ChecksumAlgorithm) (string, error) {
var size int
switch algo {
case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c:
size = 4
case types.ChecksumAlgorithmCrc64nvme, types.ChecksumAlgorithmXxhash64, types.ChecksumAlgorithmXxhash3:
size = 8
case types.ChecksumAlgorithmMd5, types.ChecksumAlgorithmXxhash128:
size = 16
case types.ChecksumAlgorithmSha1:
size = 20
case types.ChecksumAlgorithmSha256:
size = 32
case types.ChecksumAlgorithmSha512:
size = 64
default:
return "", fmt.Errorf("unsupported hash algorithm: %s", algo)
}
return base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0xff}, size)), nil
}
func processCompositeChecksum(hasher hash.Hash, checksum string) error {
data, err := base64.StdEncoding.DecodeString(checksum)
if err != nil {