mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-20 22:26:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2097 from versity/dependabot/go_modules/dev-dependencies-3789651bc5

Browse Source 
chore(deps): bump the dev-dependencies group with 7 updates
This commit is contained in:
Ben McClelland
2026-05-04 16:24:32 -07:00
committed by GitHub
parent 6731179e12 bb3cdd9cb6
commit 21da1d7e70
42 changed files with 1862 additions and 1051 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/posix/posix.go
+223 -56
View File
@@ -1659,11 +1659,79 @@ func getPartChecksum(algo types.ChecksumAlgorithm, part types.CompletedPart) str
return backend.GetStringFromPtr(part.ChecksumSHA256)
case types.ChecksumAlgorithmCrc64nvme:
return backend.GetStringFromPtr(part.ChecksumCRC64NVME)
case types.ChecksumAlgorithmSha512:
return backend.GetStringFromPtr(part.ChecksumSHA512)
case types.ChecksumAlgorithmMd5:
return backend.GetStringFromPtr(part.ChecksumMD5)
case types.ChecksumAlgorithmXxhash64:
return backend.GetStringFromPtr(part.ChecksumXXHASH64)
case types.ChecksumAlgorithmXxhash3:
return backend.GetStringFromPtr(part.ChecksumXXHASH3)
case types.ChecksumAlgorithmXxhash128:
return backend.GetStringFromPtr(part.ChecksumXXHASH128)
default:
return ""
}
}
func setStoredChecksum(checksum *s3response.Checksum, algo types.ChecksumAlgorithm, sum *string) {
if sum == nil {
return
}
switch algo {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = sum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = sum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = sum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = sum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = sum
case types.ChecksumAlgorithmSha512:
checksum.SHA512 = sum
case types.ChecksumAlgorithmMd5:
checksum.MD5 = sum
case types.ChecksumAlgorithmXxhash64:
checksum.XXHASH64 = sum
case types.ChecksumAlgorithmXxhash3:
checksum.XXHASH3 = sum
case types.ChecksumAlgorithmXxhash128:
checksum.XXHASH128 = sum
}
}
func setUploadPartChecksum(res *s3.UploadPartOutput, algo types.ChecksumAlgorithm, sum *string) {
if sum == nil {
return
}
switch algo {
case types.ChecksumAlgorithmCrc32:
res.ChecksumCRC32 = sum
case types.ChecksumAlgorithmCrc32c:
res.ChecksumCRC32C = sum
case types.ChecksumAlgorithmSha1:
res.ChecksumSHA1 = sum
case types.ChecksumAlgorithmSha256:
res.ChecksumSHA256 = sum
case types.ChecksumAlgorithmCrc64nvme:
res.ChecksumCRC64NVME = sum
case types.ChecksumAlgorithmSha512:
res.ChecksumSHA512 = sum
case types.ChecksumAlgorithmMd5:
res.ChecksumMD5 = sum
case types.ChecksumAlgorithmXxhash64:
res.ChecksumXXHASH64 = sum
case types.ChecksumAlgorithmXxhash3:
res.ChecksumXXHASH3 = sum
case types.ChecksumAlgorithmXxhash128:
res.ChecksumXXHASH128 = sum
}
}
func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
release, err := p.acquireActionSlot(ctx)
if err != nil {
@@ -1947,6 +2015,11 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
var sha1 *string
var sha256 *string
var crc64nvme *string
var sha512 *string
var md5sum *string
var xxhash64 *string
var xxhash3 *string
var xxhash128 *string
var gotSum *string
switch checksums.Algorithm {
@@ -1970,6 +2043,26 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
gotSum = input.ChecksumCRC64NVME
checksums.CRC64NVME = &value
crc64nvme = &value
case types.ChecksumAlgorithmSha512:
gotSum = input.ChecksumSHA512
checksums.SHA512 = &value
sha512 = &value
case types.ChecksumAlgorithmMd5:
gotSum = input.ChecksumMD5
checksums.MD5 = &value
md5sum = &value
case types.ChecksumAlgorithmXxhash64:
gotSum = input.ChecksumXXHASH64
checksums.XXHASH64 = &value
xxhash64 = &value
case types.ChecksumAlgorithmXxhash3:
gotSum = input.ChecksumXXHASH3
checksums.XXHASH3 = &value
xxhash3 = &value
case types.ChecksumAlgorithmXxhash128:
gotSum = input.ChecksumXXHASH128
checksums.XXHASH128 = &value
xxhash128 = &value
}
// Check if the provided checksum and the calculated one are the same.
@@ -2182,6 +2275,11 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
ChecksumSHA1: sha1,
ChecksumSHA256: sha256,
ChecksumCRC64NVME: crc64nvme,
ChecksumSHA512: sha512,
ChecksumMD5: md5sum,
ChecksumXXHASH64: xxhash64,
ChecksumXXHASH3: xxhash3,
ChecksumXXHASH128: xxhash128,
ChecksumType: &checksums.Type,
}, versionID, nil
}
@@ -2218,6 +2316,11 @@ func validatePartChecksum(checksum s3response.Checksum, part types.CompletedPart
{part.ChecksumSHA1, getString(checksum.SHA1), types.ChecksumAlgorithmSha1},
{part.ChecksumSHA256, getString(checksum.SHA256), types.ChecksumAlgorithmSha256},
{part.ChecksumCRC64NVME, getString(checksum.CRC64NVME), types.ChecksumAlgorithmCrc64nvme},
{part.ChecksumSHA512, getString(checksum.SHA512), types.ChecksumAlgorithmSha512},
{part.ChecksumMD5, getString(checksum.MD5), types.ChecksumAlgorithmMd5},
{part.ChecksumXXHASH64, getString(checksum.XXHASH64), types.ChecksumAlgorithmXxhash64},
{part.ChecksumXXHASH3, getString(checksum.XXHASH3), types.ChecksumAlgorithmXxhash3},
{part.ChecksumXXHASH128, getString(checksum.XXHASH128), types.ChecksumAlgorithmXxhash128},
} {
if cs.checksum == nil {
continue
@@ -2260,6 +2363,21 @@ func numberOfChecksums(part types.CompletedPart) int {
if getString(part.ChecksumCRC64NVME) != "" {
counter++
}
if getString(part.ChecksumSHA512) != "" {
counter++
}
if getString(part.ChecksumMD5) != "" {
counter++
}
if getString(part.ChecksumXXHASH64) != "" {
counter++
}
if getString(part.ChecksumXXHASH3) != "" {
counter++
}
if getString(part.ChecksumXXHASH128) != "" {
counter++
}
return counter
}
@@ -2784,6 +2902,11 @@ func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3resp
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
ChecksumCRC64NVME: checksum.CRC64NVME,
ChecksumSHA512: checksum.SHA512,
ChecksumMD5: checksum.MD5,
ChecksumXXHASH64: checksum.XXHASH64,
ChecksumXXHASH3: checksum.XXHASH3,
ChecksumXXHASH128: checksum.XXHASH128,
})
}
@@ -2908,6 +3031,11 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart
{input.ChecksumSHA1, utils.HashTypeSha1},
{input.ChecksumSHA256, utils.HashTypeSha256},
{input.ChecksumCRC64NVME, utils.HashTypeCRC64NVME},
{input.ChecksumSHA512, utils.HashTypeSha512},
{input.ChecksumMD5, utils.HashTypeMd5},
{input.ChecksumXXHASH64, utils.HashTypeXXHASH64},
{input.ChecksumXXHASH3, utils.HashTypeXXHASH3},
{input.ChecksumXXHASH128, utils.HashTypeXXHASH128},
}
for _, config := range hashConfigs {
@@ -3036,24 +3164,8 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart
sum = hashRdr.Sum()
}
// Assign the checksum
switch checksums.Algorithm {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = &sum
res.ChecksumCRC32 = &sum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = &sum
res.ChecksumCRC32C = &sum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = &sum
res.ChecksumSHA1 = &sum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = &sum
res.ChecksumSHA256 = &sum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = &sum
res.ChecksumCRC64NVME = &sum
}
setStoredChecksum(&checksum, checksums.Algorithm, &sum)
setUploadPartChecksum(res, checksums.Algorithm, &sum)
err := p.storeChecksums(f.File(), bucket, partPath, checksum)
if err != nil {
@@ -3095,6 +3207,16 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart
res.ChecksumSHA256 = &sumToReturn
case utils.HashTypeCRC64NVME:
res.ChecksumCRC64NVME = &sumToReturn
case utils.HashTypeSha512:
res.ChecksumSHA512 = &sumToReturn
case utils.HashTypeMd5:
res.ChecksumMD5 = &sumToReturn
case utils.HashTypeXXHASH64:
res.ChecksumXXHASH64 = &sumToReturn
case utils.HashTypeXXHASH3:
res.ChecksumXXHASH3 = &sumToReturn
case utils.HashTypeXXHASH128:
res.ChecksumXXHASH128 = &sumToReturn
}
}
}
@@ -3339,18 +3461,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
}
sum := hashRdr.Sum()
switch algo {
case types.ChecksumAlgorithmCrc32:
checksums.CRC32 = &sum
case types.ChecksumAlgorithmCrc32c:
checksums.CRC32C = &sum
case types.ChecksumAlgorithmSha1:
checksums.SHA1 = &sum
case types.ChecksumAlgorithmSha256:
checksums.SHA256 = &sum
case types.ChecksumAlgorithmCrc64nvme:
checksums.CRC64NVME = &sum
}
setStoredChecksum(&checksums, algo, &sum)
err := p.storeChecksums(f.File(), *upi.Bucket, partPath, checksums)
if err != nil {
@@ -3392,6 +3503,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumSHA512: checksums.SHA512,
ChecksumMD5: checksums.MD5,
ChecksumXXHASH64: checksums.XXHASH64,
ChecksumXXHASH3: checksums.XXHASH3,
ChecksumXXHASH128: checksums.XXHASH128,
}, nil
}
@@ -3407,6 +3523,16 @@ func getEmptyChecksumValue(algo types.ChecksumAlgorithm) string {
return "2jmj7l5rSw0yVb/vlWAYkK/YBwk="
case types.ChecksumAlgorithmSha256:
return "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
case types.ChecksumAlgorithmSha512:
return "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="
case types.ChecksumAlgorithmMd5:
return "1B2M2Y8AsgTpgAmY7PhCfg=="
case types.ChecksumAlgorithmXxhash64:
return "70bbN1HY6Zk="
case types.ChecksumAlgorithmXxhash3:
return "LQaABTjTlMI="
case types.ChecksumAlgorithmXxhash128:
return "maoG0wFHmNhgAcMkRo1Jfw=="
default:
// default to crc64nvme
return "AAAAAAAAAAA="
@@ -3482,6 +3608,11 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
{po.ChecksumSHA1, utils.HashTypeSha1},
{po.ChecksumSHA256, utils.HashTypeSha256},
{po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME},
{po.ChecksumSHA512, utils.HashTypeSha512},
{po.ChecksumMD5, utils.HashTypeMd5},
{po.ChecksumXXHASH64, utils.HashTypeXXHASH64},
{po.ChecksumXXHASH3, utils.HashTypeXXHASH3},
{po.ChecksumXXHASH128, utils.HashTypeXXHASH128},
}
for _, config := range hashConfigs {
@@ -3556,19 +3687,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
Algorithm: checksumAlgorithm,
}
// Store the calculated checksum in the object metadata
switch checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = &expectedSum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = &expectedSum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = &expectedSum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = &expectedSum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = &expectedSum
}
setStoredChecksum(&checksum, checksumAlgorithm, &expectedSum)
err = p.storeChecksums(nil, *po.Bucket, *po.Key, checksum)
if err != nil {
@@ -3585,6 +3704,11 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
ChecksumCRC64NVME: checksum.CRC64NVME,
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
ChecksumSHA512: checksum.SHA512,
ChecksumMD5: checksum.MD5,
ChecksumXXHASH64: checksum.XXHASH64,
ChecksumXXHASH3: checksum.XXHASH3,
ChecksumXXHASH128: checksum.XXHASH128,
}, nil
}
@@ -3736,19 +3860,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
Algorithm: checksumAlgorithm,
}
// Store the calculated checksum in the object metadata
switch checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = &sum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = &sum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = &sum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = &sum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = &sum
}
setStoredChecksum(&checksum, checksumAlgorithm, &sum)
err = p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
@@ -3849,6 +3961,11 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
ChecksumCRC64NVME: checksum.CRC64NVME,
ChecksumSHA512: checksum.SHA512,
ChecksumMD5: checksum.MD5,
ChecksumXXHASH64: checksum.XXHASH64,
ChecksumXXHASH3: checksum.XXHASH3,
ChecksumXXHASH128: checksum.XXHASH128,
Size: &objsize,
ChecksumType: checksum.Type,
}, nil
@@ -4450,6 +4567,11 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumSHA512: checksums.SHA512,
ChecksumMD5: checksums.MD5,
ChecksumXXHASH64: checksums.XXHASH64,
ChecksumXXHASH3: checksums.XXHASH3,
ChecksumXXHASH128: checksums.XXHASH128,
ChecksumType: checksums.Type,
AcceptRanges: backend.GetPtrFromString("bytes"),
ContentLength: &length,
@@ -4613,6 +4735,11 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumSHA512: checksums.SHA512,
ChecksumMD5: checksums.MD5,
ChecksumXXHASH64: checksums.XXHASH64,
ChecksumXXHASH3: checksums.XXHASH3,
ChecksumXXHASH128: checksums.XXHASH128,
ChecksumType: checksums.Type,
PartsCount: partsCount,
}, nil
@@ -4864,6 +4991,11 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumSHA512: checksums.SHA512,
ChecksumMD5: checksums.MD5,
ChecksumXXHASH64: checksums.XXHASH64,
ChecksumXXHASH3: checksums.XXHASH3,
ChecksumXXHASH128: checksums.XXHASH128,
ChecksumType: checksums.Type,
TagCount: tagCount,
PartsCount: partsCount,
@@ -4907,6 +5039,11 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr
ChecksumSHA1: data.ChecksumSHA1,
ChecksumSHA256: data.ChecksumSHA256,
ChecksumCRC64NVME: data.ChecksumCRC64NVME,
ChecksumSHA512: data.ChecksumSHA512,
ChecksumMD5: data.ChecksumMD5,
ChecksumXXHASH64: data.ChecksumXXHASH64,
ChecksumXXHASH3: data.ChecksumXXHASH3,
ChecksumXXHASH128: data.ChecksumXXHASH128,
ChecksumType: data.ChecksumType,
},
}, nil
@@ -5053,6 +5190,11 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
var sha1 *string
var sha256 *string
var crc64nvme *string
var sha512 *string
var md5sum *string
var xxhash64 *string
var xxhash3 *string
var xxhash128 *string
var chType types.ChecksumType
dstObjdPath := joinPathWithTrailer(dstBucket, dstObject)
@@ -5113,6 +5255,21 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
case utils.HashTypeCRC64NVME:
checksums.CRC64NVME = &sum
crc64nvme = &sum
case utils.HashTypeSha512:
checksums.SHA512 = &sum
sha512 = &sum
case utils.HashTypeMd5:
checksums.MD5 = &sum
md5sum = &sum
case utils.HashTypeXXHASH64:
checksums.XXHASH64 = &sum
xxhash64 = &sum
case utils.HashTypeXXHASH3:
checksums.XXHASH3 = &sum
xxhash3 = &sum
case utils.HashTypeXXHASH128:
checksums.XXHASH128 = &sum
xxhash128 = &sum
}
// If a new checksum is calculated, the checksum type
@@ -5236,6 +5393,11 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
sha1 = res.ChecksumSHA1
sha256 = res.ChecksumSHA256
crc64nvme = res.ChecksumCRC64NVME
sha512 = res.ChecksumSHA512
md5sum = res.ChecksumMD5
xxhash64 = res.ChecksumXXHASH64
xxhash3 = res.ChecksumXXHASH3
xxhash128 = res.ChecksumXXHASH128
chType = res.ChecksumType
}
@@ -5253,6 +5415,11 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
ChecksumSHA1: sha1,
ChecksumSHA256: sha256,
ChecksumCRC64NVME: crc64nvme,
ChecksumSHA512: sha512,
ChecksumMD5: md5sum,
ChecksumXXHASH64: xxhash64,
ChecksumXXHASH3: xxhash3,
ChecksumXXHASH128: xxhash128,
ChecksumType: chType,
},
VersionId: version,
backend/s3proxy/s3.go
+75
View File
@@ -483,6 +483,21 @@ func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
input.ChecksumSHA256 = nil
}
if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" {
input.ChecksumSHA512 = nil
}
if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" {
input.ChecksumMD5 = nil
}
if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" {
input.ChecksumXXHASH64 = nil
}
if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" {
input.ChecksumXXHASH3 = nil
}
if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" {
input.ChecksumXXHASH128 = nil
}
if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
input.ExpectedBucketOwner = nil
}
@@ -518,6 +533,11 @@ func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
ChecksumCRC64NVME: out.ChecksumCRC64NVME,
ChecksumSHA1: out.ChecksumSHA1,
ChecksumSHA256: out.ChecksumSHA256,
ChecksumSHA512: out.ChecksumSHA512,
ChecksumMD5: out.ChecksumMD5,
ChecksumXXHASH64: out.ChecksumXXHASH64,
ChecksumXXHASH3: out.ChecksumXXHASH3,
ChecksumXXHASH128: out.ChecksumXXHASH128,
ChecksumType: &out.ChecksumType,
}
if out.VersionId != nil {
@@ -653,6 +673,11 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
ChecksumCRC64NVME: p.ChecksumCRC64NVME,
ChecksumSHA1: p.ChecksumSHA1,
ChecksumSHA256: p.ChecksumSHA256,
ChecksumSHA512: p.ChecksumSHA512,
ChecksumMD5: p.ChecksumMD5,
ChecksumXXHASH64: p.ChecksumXXHASH64,
ChecksumXXHASH3: p.ChecksumXXHASH3,
ChecksumXXHASH128: p.ChecksumXXHASH128,
})
}
pnm, err := strconv.Atoi(*output.PartNumberMarker)
@@ -709,6 +734,21 @@ func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
input.ChecksumSHA256 = nil
}
if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" {
input.ChecksumSHA512 = nil
}
if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" {
input.ChecksumMD5 = nil
}
if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" {
input.ChecksumXXHASH64 = nil
}
if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" {
input.ChecksumXXHASH3 = nil
}
if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" {
input.ChecksumXXHASH128 = nil
}
if input.ContentMD5 != nil && *input.ContentMD5 == "" {
input.ContentMD5 = nil
}
@@ -790,6 +830,11 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME,
ChecksumSHA1: output.CopyPartResult.ChecksumSHA1,
ChecksumSHA256: output.CopyPartResult.ChecksumSHA256,
ChecksumSHA512: output.CopyPartResult.ChecksumSHA512,
ChecksumMD5: output.CopyPartResult.ChecksumMD5,
ChecksumXXHASH64: output.CopyPartResult.ChecksumXXHASH64,
ChecksumXXHASH3: output.CopyPartResult.ChecksumXXHASH3,
ChecksumXXHASH128: output.CopyPartResult.ChecksumXXHASH128,
}, nil
}
@@ -815,6 +860,21 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
input.ChecksumSHA256 = nil
}
if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" {
input.ChecksumSHA512 = nil
}
if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" {
input.ChecksumMD5 = nil
}
if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" {
input.ChecksumXXHASH64 = nil
}
if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" {
input.ChecksumXXHASH3 = nil
}
if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" {
input.ChecksumXXHASH128 = nil
}
if input.ContentDisposition != nil && *input.ContentDisposition == "" {
input.ContentDisposition = nil
}
@@ -913,6 +973,11 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
ChecksumSHA1: input.ChecksumSHA1,
ChecksumSHA256: input.ChecksumSHA256,
ChecksumCRC64NVME: input.ChecksumCRC64NVME,
ChecksumSHA512: input.ChecksumSHA512,
ChecksumMD5: input.ChecksumMD5,
ChecksumXXHASH64: input.ChecksumXXHASH64,
ChecksumXXHASH3: input.ChecksumXXHASH3,
ChecksumXXHASH128: input.ChecksumXXHASH128,
ContentMD5: input.ContentMD5,
ExpectedBucketOwner: input.ExpectedBucketOwner,
GrantFullControl: input.GrantFullControl,
@@ -947,6 +1012,11 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
ChecksumCRC64NVME: output.ChecksumCRC64NVME,
ChecksumSHA1: output.ChecksumSHA1,
ChecksumSHA256: output.ChecksumSHA256,
ChecksumSHA512: output.ChecksumSHA512,
ChecksumMD5: output.ChecksumMD5,
ChecksumXXHASH64: output.ChecksumXXHASH64,
ChecksumXXHASH3: output.ChecksumXXHASH3,
ChecksumXXHASH128: output.ChecksumXXHASH128,
Size: output.Size,
}, nil
}
@@ -1292,6 +1362,11 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input s3response.CopyObjectInp
ChecksumCRC64NVME: out.CopyObjectResult.ChecksumCRC64NVME,
ChecksumSHA1: out.CopyObjectResult.ChecksumSHA1,
ChecksumSHA256: out.CopyObjectResult.ChecksumSHA256,
ChecksumSHA512: out.CopyObjectResult.ChecksumSHA512,
ChecksumMD5: out.CopyObjectResult.ChecksumMD5,
ChecksumXXHASH64: out.CopyObjectResult.ChecksumXXHASH64,
ChecksumXXHASH3: out.CopyObjectResult.ChecksumXXHASH3,
ChecksumXXHASH128: out.CopyObjectResult.ChecksumXXHASH128,
ChecksumType: out.CopyObjectResult.ChecksumType,
ETag: out.CopyObjectResult.ETag,
LastModified: out.CopyObjectResult.LastModified,
cmd/versitygw/test.go
+19
View File
@@ -38,6 +38,7 @@ var (
checksumDisable bool
versioningEnabled bool
azureTests bool
sidecarTests bool
tlsStatus bool
parallel bool
)
@@ -116,6 +117,12 @@ func initTestCommands() []*cli.Command {
Destination: &azureTests,
Aliases: []string{"azure"},
},
&cli.BoolFlag{
Name: "sidecar-test-mode",
Usage: "Skips tests that are not supported by Sidecar",
Destination: &sidecarTests,
Aliases: []string{"sidecar"},
},
&cli.BoolFlag{
Name: "parallel",
Usage: "executes the tests concurrently",
@@ -336,6 +343,9 @@ func getAction(tf testFunc) func(ctx *cli.Context) error {
if azureTests {
opts = append(opts, integration.WithAzureMode())
}
if sidecarTests {
opts = append(opts, integration.WithSidecarMode())
}
if hostStyle {
opts = append(opts, integration.WithHostStyle())
}
@@ -382,6 +392,9 @@ func extractIntTests() (commands []*cli.Command) {
if azureTests {
opts = append(opts, integration.WithAzureMode())
}
if sidecarTests {
opts = append(opts, integration.WithSidecarMode())
}
s := integration.NewS3Conf(opts...)
err := testFunc(s)
@@ -400,6 +413,12 @@ func extractIntTests() (commands []*cli.Command) {
Destination: &azureTests,
Aliases: []string{"azure"},
},
&cli.BoolFlag{
Name: "sidecar-test-mode",
Usage: "Skips tests that are not supported by Sidecar",
Destination: &sidecarTests,
Aliases: []string{"sidecar"},
},
},
})
}
go.mod
+9 -7
View File
@@ -10,12 +10,13 @@ require (
github.com/aws/aws-sdk-go-v2 v1.41.6
github.com/aws/aws-sdk-go-v2/config v1.32.16
github.com/aws/aws-sdk-go-v2/credentials v1.19.15
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.17
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1
github.com/aws/smithy-go v1.25.0
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.18
github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0
github.com/aws/smithy-go v1.25.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/davecgh/go-spew v1.1.1
github.com/go-ldap/ldap/v3 v3.4.13
github.com/gofiber/fiber/v2 v2.52.12
github.com/gofiber/fiber/v2 v2.52.13
github.com/google/go-cmp v0.7.0
github.com/google/uuid v1.6.0
github.com/hashicorp/vault-client-go v0.4.3
@@ -23,13 +24,14 @@ require (
github.com/nats-io/nats.go v1.51.0
github.com/oklog/ulid/v2 v2.1.1
github.com/pkg/xattr v0.4.12
github.com/rabbitmq/amqp091-go v1.10.0
github.com/segmentio/kafka-go v0.4.50
github.com/rabbitmq/amqp091-go v1.11.0
github.com/segmentio/kafka-go v0.4.51
github.com/smira/go-statsd v1.3.4
github.com/stretchr/testify v1.11.1
github.com/urfave/cli/v2 v2.27.7
github.com/valyala/fasthttp v1.70.0
github.com/versity/scoutfs-go v0.0.0-20240625221833-95fd765b760b
github.com/zeebo/xxh3 v1.1.0
golang.org/x/sync v0.20.0
golang.org/x/sys v0.43.0
gopkg.in/yaml.v3 v3.0.1
@@ -66,7 +68,7 @@ require (
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.21 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect
github.com/mattn/go-runewidth v0.0.23 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/nats-io/nkeys v0.4.15 // indirect
go.sum
+20 -14
View File
@@ -35,8 +35,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.15 h1:fyvgWTszojq8hEnMi8PPBTvZdTt
github.com/aws/aws-sdk-go-v2/credentials v1.19.15/go.mod h1:gJiYyMOjNg8OEdRWOf3CrFQxM2a98qmrtjx1zuiQfB8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22 h1:IOGsJ1xVWhsi+ZO7/NW8OuZZBtMJLZbk4P5HDjJO0jQ=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22/go.mod h1:b+hYdbU+jGKfXE8kKM6g1+h+L/Go3vMvzlxBsiuGsxg=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.17 h1:95y7/EqethAhFwMKJ9cDutzBhsS1h8uBwkJ5rp8pNTU=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.17/go.mod h1:77baheqr62SkTw77HWH8qpdWTd2gXKN0xg0qLvDSkpk=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.18 h1:RtBqcC84feV8kTqFVq1uX9q+Sd9bL9jctEWjF02Tss0=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.18/go.mod h1:B5OSVaJ4Qqvpt6eZTxjFejGOuW8RpMV0IRm+4x+Ksls=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 h1:GmLa5Kw1ESqtFpXsx5MmC84QWa/ZrLZvlJGa2y+4kcQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22/go.mod h1:6sW9iWm9DK9YRpRGga/qzrzNLgKpT2cIxb7Vo2eNOp0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22 h1:dY4kWZiSaXIzxnKlj17nHnBcXXBfac6UlsAx2qL6XrU=
@@ -51,8 +51,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 h1:PUmZeJU6
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22/go.mod h1:nO6egFBoAaoXze24a2C0NjQCvdpk8OueRoYimvEB9jo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22 h1:SE+aQ4DEqG53RRCAIHlCf//B2ycxGH7jFkpnAh/kKPM=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22/go.mod h1:ES3ynECd7fYeJIL6+oax+uIEljmfps0S70BaQzbMd/o=
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1 h1:kU/eBN5+MWNo/LcbNa4hWDdN76hdcd7hocU5kvu7IsU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1/go.mod h1:Fw9aqhJicIVee1VytBBjH+l+5ov6/PhbtIK/u3rt/ls=
github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0 h1:7G26Sae6PMKn4kMcU5JzNfrm1YrKwyOhowXPYR2WiWY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0/go.mod h1:Fw9aqhJicIVee1VytBBjH+l+5ov6/PhbtIK/u3rt/ls=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.10 h1:a1Fq/KXn75wSzoJaPQTgZO0wHGqE9mjFnylnqEPTchA=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.10/go.mod h1:p6+MXNxW7IA6dMgHfTAzljuwSKD0NCm/4lbS4t6+7vI=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.16 h1:x6bKbmDhsgSZwv6q19wY/u3rLk/3FGjJWyqKcIRufpE=
@@ -61,8 +61,10 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20 h1:oK/njaL8GtyEihkWMD4k3Vg
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20/go.mod h1:JHs8/y1f3zY7U5WcuzoJ/yAYGYtNIVPKLIbp61euvmg=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.0 h1:ks8KBcZPh3PYISr5dAiXCM5/Thcuxk8l+PG4+A0exds=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.0/go.mod h1:pFw33T0WLvXU3rw1WBkpMlkgIn54eCB5FYLhjDc9Foo=
github.com/aws/smithy-go v1.25.0 h1:Sz/XJ64rwuiKtB6j98nDIPyYrV1nVNJ4YU74gttcl5U=
github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
@@ -76,8 +78,8 @@ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-ldap/ldap/v3 v3.4.13 h1:+x1nG9h+MZN7h/lUi5Q3UZ0fJ1GyDQYbPvbuH38baDQ=
github.com/go-ldap/ldap/v3 v3.4.13/go.mod h1:LxsGZV6vbaK0sIvYfsv47rfh4ca0JXokCoKjZxsszv0=
github.com/gofiber/fiber/v2 v2.52.12 h1:0LdToKclcPOj8PktUdIKo9BUohjjwfnQl42Dhw8/WUw=
github.com/gofiber/fiber/v2 v2.52.12/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
github.com/gofiber/fiber/v2 v2.52.13 h1:TOKP64iqC9b5P49VrBW5tHhUOvDyrtJ0xePEfzJbCbk=
github.com/gofiber/fiber/v2 v2.52.13/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
@@ -125,8 +127,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw=
github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
@@ -151,16 +153,16 @@ github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/segmentio/kafka-go v0.4.50 h1:mcyC3tT5WeyWzrFbd6O374t+hmcu1NKt2Pu1L3QaXmc=
github.com/segmentio/kafka-go v0.4.50/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
github.com/segmentio/kafka-go v0.4.51 h1:JgDPPG75tC1rWIS2Me6MwcvXJ6f49UQ4HjAOef71Hno=
github.com/segmentio/kafka-go v0.4.51/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
@@ -194,6 +196,10 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
runtests.sh
+6 -4
View File
@@ -10,10 +10,12 @@ done
# build sidecar flag for versitygw invocations
SIDECAR_FLAG=""
SIDECAR_TEST_FLAG=""
if $USE_SIDECAR; then
rm -rf /tmp/sidecar
mkdir /tmp/sidecar
SIDECAR_FLAG="--sidecar /tmp/sidecar"
SIDECAR_TEST_FLAG="--sidecar"
fi
# make temp dirs
@@ -55,7 +57,7 @@ fi
# run tests
# full flow tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --parallel; then
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --parallel $$SIDECAR_TEST_FLAG; then
echo "full flow tests failed"
kill $GW_PID
exit 1
@@ -92,7 +94,7 @@ fi
# run tests
# full flow tests
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 full-flow --parallel; then
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 full-flow --parallel $$SIDECAR_TEST_FLAG; then
echo "full flow tests failed"
kill $GW_HTTPS_PID
exit 1
@@ -129,7 +131,7 @@ fi
# run tests
# full flow tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 full-flow -vs --parallel; then
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 full-flow -vs --parallel $$SIDECAR_TEST_FLAG; then
echo "versioning-enabled full-flow tests failed"
kill $GW_VS_PID
exit 1
@@ -161,7 +163,7 @@ fi
# run tests
# full flow tests
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 full-flow -vs --parallel; then
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 full-flow -vs --parallel $$SIDECAR_TEST_FLAG; then
echo "versioning-enabled full-flow tests failed"
kill $GW_VS_HTTPS_PID
exit 1
s3api/controllers/bucket-post.go
+10
View File
@@ -222,6 +222,11 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
})
if err != nil {
return &Response{
@@ -285,6 +290,11 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
"x-amz-version-id": utils.GetStringPtr(res.VersionID),
},
s3api/controllers/bucket-post_test.go
+5
View File
@@ -547,6 +547,11 @@ func TestS3ApiController_POSTObject(t *testing.T) {
"x-amz-checksum-crc64nvme": utils.GetStringPtr(""),
"x-amz-checksum-sha1": utils.GetStringPtr(""),
"x-amz-checksum-sha256": utils.GetStringPtr(""),
"x-amz-checksum-sha512": utils.GetStringPtr(""),
"x-amz-checksum-md5": utils.GetStringPtr(""),
"x-amz-checksum-xxhash64": utils.GetStringPtr(""),
"x-amz-checksum-xxhash3": utils.GetStringPtr(""),
"x-amz-checksum-xxhash128": utils.GetStringPtr(""),
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
"x-amz-version-id": utils.GetStringPtr("vid-123"),
},
s3api/controllers/object-get.go
+5
View File
@@ -556,6 +556,11 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"Content-Type": utils.ApplyOverride(res.ContentType, responseOverrides["Content-Type"]),
"x-amz-version-id": res.VersionId,
"Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength),
s3api/controllers/object-get_test.go
+10
View File
@@ -799,6 +799,11 @@ func TestS3ApiController_GetObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
@@ -844,6 +849,11 @@ func TestS3ApiController_GetObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
s3api/controllers/object-head.go
+5
View File
@@ -188,6 +188,11 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-version-id": res.VersionId,
"x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount),
"x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode),
s3api/controllers/object-head_test.go
+10
View File
@@ -189,6 +189,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
@@ -233,6 +238,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
s3api/controllers/object-post.go
+5
View File
@@ -375,6 +375,11 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
ChecksumType: checksumType,
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
s3api/controllers/object-put.go
+20
View File
@@ -306,6 +306,11 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
})
var headers map[string]*string
if err == nil {
@@ -316,6 +321,11 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
}
}
return &Response{
@@ -789,6 +799,11 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
})
@@ -800,6 +815,11 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
"x-amz-version-id": &res.VersionID,
"x-amz-object-size": utils.ConvertPtrToStringPtr(res.Size),
s3api/controllers/object-put_test.go
+20
View File
@@ -535,6 +535,11 @@ func TestS3ApiController_UploadPart(t *testing.T) {
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
@@ -1214,6 +1219,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-checksum-type": nil,
"x-amz-version-id": emptyStringPtr,
"x-amz-object-size": nil,
@@ -1255,6 +1265,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
ChecksumSHA1: utils.GetStringPtr("sha1"),
ChecksumSHA256: utils.GetStringPtr("sha256"),
ChecksumCRC64NVME: utils.GetStringPtr("crc64nvme"),
ChecksumSHA512: utils.GetStringPtr("sha512"),
ChecksumMD5: utils.GetStringPtr("md5"),
ChecksumXXHASH64: utils.GetStringPtr("xxhash64"),
ChecksumXXHASH3: utils.GetStringPtr("xxhash3"),
ChecksumXXHASH128: utils.GetStringPtr("xxhash128"),
ChecksumType: types.ChecksumTypeComposite,
VersionID: "versionId",
Size: &objSize,
@@ -1269,6 +1284,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
"x-amz-checksum-crc64nvme": utils.GetStringPtr("crc64nvme"),
"x-amz-checksum-sha1": utils.GetStringPtr("sha1"),
"x-amz-checksum-sha256": utils.GetStringPtr("sha256"),
"x-amz-checksum-sha512": utils.GetStringPtr("sha512"),
"x-amz-checksum-md5": utils.GetStringPtr("md5"),
"x-amz-checksum-xxhash64": utils.GetStringPtr("xxhash64"),
"x-amz-checksum-xxhash3": utils.GetStringPtr("xxhash3"),
"x-amz-checksum-xxhash128": utils.GetStringPtr("xxhash128"),
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
"x-amz-version-id": utils.GetStringPtr("versionId"),
"x-amz-object-size": utils.ConvertToStringPtr(objSize),
s3api/middlewares/checksum.go
+2 -2
View File
@@ -49,7 +49,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
var err error
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
r, err = utils.NewHashReader(r, md5sum, utils.HashTypeMd5)
r, err = utils.NewHashReader(r, md5sum, utils.HashTypeContentMD5)
return r
})
if err != nil {
@@ -70,7 +70,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
return s3err.GetAPIError(s3err.ErrInvalidDigest)
}
rdr, err = utils.NewHashReader(bytes.NewReader(body), md5sum, utils.HashTypeMd5)
rdr, err = utils.NewHashReader(bytes.NewReader(body), md5sum, utils.HashTypeContentMD5)
if err != nil {
return err
}
s3api/utils/chunk-reader.go
+11 -1
View File
@@ -79,6 +79,11 @@ const (
checksumTypeSha1 checksumType = "x-amz-checksum-sha1"
checksumTypeSha256 checksumType = "x-amz-checksum-sha256"
checksumTypeCrc64nvme checksumType = "x-amz-checksum-crc64nvme"
checksumTypeSha512 checksumType = "x-amz-checksum-sha512"
checksumTypeMd5 checksumType = "x-amz-checksum-md5"
checksumTypeXxhash64 checksumType = "x-amz-checksum-xxhash64"
checksumTypeXxhash3 checksumType = "x-amz-checksum-xxhash3"
checksumTypeXxhash128 checksumType = "x-amz-checksum-xxhash128"
)
func (c checksumType) isValid() bool {
@@ -86,7 +91,12 @@ func (c checksumType) isValid() bool {
c == checksumTypeCrc32c ||
c == checksumTypeSha1 ||
c == checksumTypeSha256 ||
c == checksumTypeCrc64nvme
c == checksumTypeCrc64nvme ||
c == checksumTypeSha512 ||
c == checksumTypeMd5 ||
c == checksumTypeXxhash64 ||
c == checksumTypeXxhash3 ||
c == checksumTypeXxhash128
}
// Extracts and validates the checksum type from the 'X-Amz-Trailer' header
s3api/utils/csum-reader.go
+75 -6
View File
@@ -18,6 +18,7 @@ import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"errors"
@@ -28,14 +29,18 @@ import (
"io"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
)
// HashType identifies the checksum algorithm to be used
type HashType string
const (
// HashTypeMd5 generates MD5 checksum for the data stream
// HashTypeContentMD5 generates MD5 checksum for the Content-MD5 header.
HashTypeContentMD5 HashType = "content-md5"
// HashTypeMd5 generates MD5 Base64-Encoded checksum for the data stream
HashTypeMd5 HashType = "md5"
// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
HashTypeSha256 HashType = "sha256"
@@ -43,12 +48,20 @@ const (
HashTypeSha256Hex HashType = "sha256-hex"
// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
HashTypeSha1 HashType = "sha1"
// HashTypeSha512 generates SHA512 Base64-Encoded checksum for the data stream
HashTypeSha512 HashType = "sha512"
// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
HashTypeCRC32 HashType = "crc32"
// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
HashTypeCRC32C HashType = "crc32c"
// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
HashTypeCRC64NVME HashType = "crc64nvme"
// HashTypeXXHASH64 generates XXHASH64 Base64-Encoded checksum for the data stream
HashTypeXXHASH64 HashType = "xxhash64"
// HashTypeXXHASH3 generates XXHASH3 Base64-Encoded checksum for the data stream
HashTypeXXHASH3 HashType = "xxhash3"
// HashTypeXXHASH128 generates XXHASH128 Base64-Encoded checksum for the data stream
HashTypeXXHASH128 HashType = "xxhash128"
// HashTypeNone is a no-op checksum for the data stream
HashTypeNone HashType = "none"
)
@@ -75,7 +88,7 @@ var (
func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) {
var hash hash.Hash
switch ht {
case HashTypeMd5:
case HashTypeContentMD5, HashTypeMd5:
hash = md5.New()
case HashTypeSha256Hex:
hash = sha256.New()
@@ -83,12 +96,20 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
hash = sha256.New()
case HashTypeSha1:
hash = sha1.New()
case HashTypeSha512:
hash = sha512.New()
case HashTypeCRC32:
hash = crc32.NewIEEE()
case HashTypeCRC32C:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeCRC64NVME:
hash = crc64.New(crc64NVMETable)
case HashTypeXXHASH64:
hash = xxhash.New()
case HashTypeXXHASH3:
hash = xxh3.New()
case HashTypeXXHASH128:
hash = xxh3.New128()
case HashTypeNone:
hash = noop{}
default:
@@ -112,11 +133,16 @@ func (hr *HashReader) Read(p []byte) (int, error) {
}
if errors.Is(readerr, io.EOF) && hr.sum != "" {
switch hr.hashType {
case HashTypeMd5:
case HashTypeContentMD5:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrBadDigest)
}
case HashTypeMd5:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmMd5)
}
case HashTypeSha256Hex:
sum := hr.Sum()
if sum != hr.sum {
@@ -142,11 +168,31 @@ func (hr *HashReader) Read(p []byte) (int, error) {
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
}
case HashTypeSha512:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha512)
}
case HashTypeCRC64NVME:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
}
case HashTypeXXHASH64:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash64)
}
case HashTypeXXHASH3:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash3)
}
case HashTypeXXHASH128:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash128)
}
default:
return n, errInvalidHashType
}
@@ -161,7 +207,7 @@ func (hr *HashReader) SetReader(r io.Reader) {
// Sum returns the checksum hash of the data read so far
func (hr *HashReader) Sum() string {
switch hr.hashType {
case HashTypeMd5:
case HashTypeContentMD5, HashTypeMd5:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256Hex:
return hex.EncodeToString(hr.hash.Sum(nil))
@@ -173,8 +219,16 @@ func (hr *HashReader) Sum() string {
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha512:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeCRC64NVME:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH64:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH3:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH128:
return Base64SumString(hr.hash.Sum(nil))
default:
return ""
}
@@ -197,8 +251,8 @@ func (n noop) Reset() {}
func (n noop) Size() int { return 0 }
func (n noop) BlockSize() int { return 1 }
// IsChecksumComposable tests if the final foll object crc can be calculated
// based on the part crc values.
// IsChecksumComposable tests if the final full-object crc can be calculated
// from the part crc values.
func IsChecksumComposable(algo types.ChecksumAlgorithm) bool {
switch algo {
case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c, types.ChecksumAlgorithmCrc64nvme:
@@ -309,17 +363,32 @@ func AddCRCChecksum(algo types.ChecksumAlgorithm, crc, partCrc string, partLen i
// - CRC32C
// - SHA1
// - SHA256
// - SHA512
// - MD5
// - XXHASH64
// - XXHASH3
// - XXHASH128
func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
var hasher hash.Hash
switch ht {
case HashTypeMd5:
hasher = md5.New()
case HashTypeSha256:
hasher = sha256.New()
case HashTypeSha1:
hasher = sha1.New()
case HashTypeSha512:
hasher = sha512.New()
case HashTypeCRC32:
hasher = crc32.NewIEEE()
case HashTypeCRC32C:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeXXHASH64:
hasher = xxhash.New()
case HashTypeXXHASH3:
hasher = xxh3.New()
case HashTypeXXHASH128:
hasher = xxh3.New128()
case HashTypeNone:
hasher = noop{}
default:
s3api/utils/csum-reader_test.go
+99
View File
@@ -15,12 +15,22 @@
package utils
import (
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/stretchr/testify/assert"
"github.com/zeebo/xxh3"
)
func TestAddCRCChecksum_CRC32(t *testing.T) {
@@ -118,3 +128,92 @@ func TestAddCRCChecksum_CRC64NVME(t *testing.T) {
t.Errorf("CRC64NVME combine mismatch: got %x, want %x", combinedVal, crcFull)
}
}
func base64HashForTest(h hash.Hash, data []byte) string {
h.Write(data)
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func TestNewHashReader_NewChecksumAlgorithms(t *testing.T) {
data := []byte("checksum payload")
tests := []struct {
name string
hashType HashType
hasher hash.Hash
}{
{name: "md5", hashType: HashTypeMd5, hasher: md5.New()},
{name: "sha1", hashType: HashTypeSha1, hasher: sha1.New()},
{name: "sha256", hashType: HashTypeSha256, hasher: sha256.New()},
{name: "sha512", hashType: HashTypeSha512, hasher: sha512.New()},
{name: "crc32", hashType: HashTypeCRC32, hasher: crc32.NewIEEE()},
{name: "crc32c", hashType: HashTypeCRC32C, hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
{name: "crc64nvme", hashType: HashTypeCRC64NVME, hasher: crc64.New(crc64NVMETable)},
{name: "xxhash64", hashType: HashTypeXXHASH64, hasher: xxhash.New()},
{name: "xxhash3", hashType: HashTypeXXHASH3, hasher: xxh3.New()},
{name: "xxhash128", hashType: HashTypeXXHASH128, hasher: xxh3.New128()},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
expected := base64HashForTest(tt.hasher, data)
rdr, err := NewHashReader(bytes.NewReader(data), expected, tt.hashType)
if !assert.NoError(t, err) {
return
}
_, err = io.Copy(io.Discard, rdr)
assert.NoError(t, err)
assert.Equal(t, expected, rdr.Sum())
})
}
}
func TestNewCompositeChecksumReader_NewChecksumAlgorithms(t *testing.T) {
part1 := []byte("part one")
part2 := []byte("part two")
tests := []struct {
name string
hashType HashType
newHasher func() hash.Hash
}{
{name: "md5", hashType: HashTypeMd5, newHasher: md5.New},
{name: "sha1", hashType: HashTypeSha1, newHasher: sha1.New},
{name: "sha256", hashType: HashTypeSha256, newHasher: sha256.New},
{name: "sha512", hashType: HashTypeSha512, newHasher: sha512.New},
{name: "crc32", hashType: HashTypeCRC32, newHasher: func() hash.Hash { return crc32.NewIEEE() }},
{name: "crc32c", hashType: HashTypeCRC32C, newHasher: func() hash.Hash { return crc32.New(crc32.MakeTable(crc32.Castagnoli)) }},
{name: "xxhash64", hashType: HashTypeXXHASH64, newHasher: func() hash.Hash { return xxhash.New() }},
{name: "xxhash3", hashType: HashTypeXXHASH3, newHasher: func() hash.Hash { return xxh3.New() }},
{name: "xxhash128", hashType: HashTypeXXHASH128, newHasher: func() hash.Hash { return xxh3.New128() }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
part1Sum := base64HashForTest(tt.newHasher(), part1)
part2Sum := base64HashForTest(tt.newHasher(), part2)
composite, err := NewCompositeChecksumReader(tt.hashType)
if !assert.NoError(t, err) {
return
}
assert.NoError(t, composite.Process(part1Sum))
assert.NoError(t, composite.Process(part2Sum))
part1Bytes, err := base64.StdEncoding.DecodeString(part1Sum)
if !assert.NoError(t, err) {
return
}
part2Bytes, err := base64.StdEncoding.DecodeString(part2Sum)
if !assert.NoError(t, err) {
return
}
expectedHasher := tt.newHasher()
expectedHasher.Write(part1Bytes)
expectedHasher.Write(part2Bytes)
expected := base64.StdEncoding.EncodeToString(expectedHasher.Sum(nil))
assert.Equal(t, expected, composite.Sum())
})
}
}
s3api/utils/unsigned-chunk-reader.go
+14
View File
@@ -17,8 +17,10 @@ package utils
import (
"bufio"
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"errors"
"hash"
@@ -30,8 +32,10 @@ import (
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
)
var (
@@ -499,6 +503,16 @@ func getHasher(ct checksumType) (hash.Hash, error) {
return sha1.New(), nil
case checksumTypeSha256:
return sha256.New(), nil
case checksumTypeSha512:
return sha512.New(), nil
case checksumTypeMd5:
return md5.New(), nil
case checksumTypeXxhash64:
return xxhash.New(), nil
case checksumTypeXxhash3:
return xxh3.New(), nil
case checksumTypeXxhash128:
return xxh3.New128(), nil
default:
return nil, errors.New("unsupported checksum type")
}
s3api/utils/utils.go
+31 -1
View File
@@ -721,8 +721,13 @@ var checksumLengths = map[types.ChecksumAlgorithm]int{
types.ChecksumAlgorithmCrc32: 4,
types.ChecksumAlgorithmCrc32c: 4,
types.ChecksumAlgorithmCrc64nvme: 8,
types.ChecksumAlgorithmMd5: 16,
types.ChecksumAlgorithmSha1: 20,
types.ChecksumAlgorithmSha256: 32,
types.ChecksumAlgorithmSha512: 64,
types.ChecksumAlgorithmXxhash64: 8,
types.ChecksumAlgorithmXxhash3: 8,
types.ChecksumAlgorithmXxhash128: 16,
}
func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
@@ -753,7 +758,12 @@ func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
alg != types.ChecksumAlgorithmCrc32c &&
alg != types.ChecksumAlgorithmSha1 &&
alg != types.ChecksumAlgorithmSha256 &&
alg != types.ChecksumAlgorithmCrc64nvme {
alg != types.ChecksumAlgorithmCrc64nvme &&
alg != types.ChecksumAlgorithmSha512 &&
alg != types.ChecksumAlgorithmMd5 &&
alg != types.ChecksumAlgorithmXxhash64 &&
alg != types.ChecksumAlgorithmXxhash3 &&
alg != types.ChecksumAlgorithmXxhash128 {
debuglogger.Logf("invalid checksum algorithm: %v\n", alg)
return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
}
@@ -799,6 +809,26 @@ var checksumMap checksumSchema = checksumSchema{
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmSha512: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmMd5: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash64: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash3: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash128: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
// Both could be empty
"": checksumTypeSchema{
"": struct{}{},
s3api/utils/utils_test.go
+211
View File
@@ -707,6 +707,41 @@ func TestIsChecksumAlgorithmValid(t *testing.T) {
},
wantErr: false,
},
{
name: "sha512",
args: args{
alg: types.ChecksumAlgorithmSha512,
},
wantErr: false,
},
{
name: "md5",
args: args{
alg: types.ChecksumAlgorithmMd5,
},
wantErr: false,
},
{
name: "xxhash64",
args: args{
alg: types.ChecksumAlgorithmXxhash64,
},
wantErr: false,
},
{
name: "xxhash3",
args: args{
alg: types.ChecksumAlgorithmXxhash3,
},
wantErr: false,
},
{
name: "xxhash128",
args: args{
alg: types.ChecksumAlgorithmXxhash128,
},
wantErr: false,
},
{
name: "invalid",
args: args{
@@ -806,6 +841,102 @@ func TestIsValidChecksum(t *testing.T) {
},
want: true,
},
{
name: "invalid-crc64nvme",
args: args{
checksum: "ww2FVQ==",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: false,
},
{
name: "valid-crc64nvme",
args: args{
checksum: "AAAAAAAAAAA=",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: true,
},
{
name: "invalid-sha512",
args: args{
checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
algorithm: types.ChecksumAlgorithmSha512,
},
want: false,
},
{
name: "valid-sha512",
args: args{
checksum: "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==",
algorithm: types.ChecksumAlgorithmSha512,
},
want: true,
},
{
name: "invalid-md5",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmMd5,
},
want: false,
},
{
name: "valid-md5",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmMd5,
},
want: true,
},
{
name: "invalid-xxhash64",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: false,
},
{
name: "valid-xxhash64",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: true,
},
{
name: "invalid-xxhash3",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: false,
},
{
name: "valid-xxhash3",
args: args{
checksum: "LQaABTjTlMI=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: true,
},
{
name: "invalid-xxhash128",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: false,
},
{
name: "valid-xxhash128",
args: args{
checksum: "maoG0wFHmNhgAcMkRo1Jfw==",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -1090,6 +1221,86 @@ func Test_checkChecksumTypeAndAlgo(t *testing.T) {
},
wantErr: true,
},
{
name: "full_object-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "composite-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-empty",
args: args{
s3err/s3err.go
+1 -1
View File
@@ -764,7 +764,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrInvalidChecksumAlgorithm: {
Code: "InvalidRequest",
Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]",
Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, CRC64NVME, MD5, SHA1, SHA256, SHA512, XXHASH128, XXHASH3, XXHASH64]",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidChecksumPart: {
s3response/s3response.go
+35
View File
@@ -37,6 +37,11 @@ type PutObjectOutput struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
Size *int64
ChecksumType types.ChecksumType
}
@@ -52,6 +57,11 @@ type Part struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
}
func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -375,6 +385,11 @@ type CopyObjectResult struct {
ChecksumCRC64NVME *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
ChecksumType types.ChecksumType
ETag *string
LastModified *time.Time
@@ -404,6 +419,11 @@ type CopyPartResult struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
// not included in the body
CopySourceVersionId string `xml:"-"`
@@ -439,6 +459,11 @@ type CompleteMultipartUploadResult struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
ChecksumType *types.ChecksumType
}
@@ -559,6 +584,11 @@ type PutObjectInput struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
ContentMD5 *string
ExpectedBucketOwner *string
GrantFullControl *string
@@ -731,6 +761,11 @@ type Checksum struct {
SHA1 *string
SHA256 *string
CRC64NVME *string
SHA512 *string
MD5 *string
XXHASH64 *string
XXHASH3 *string
XXHASH128 *string
}
// LocationConstraint represents the GetBucketLocation response
tests/integration/CompleteMultipartUpload.go
+107 -109
View File
@@ -518,6 +518,26 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error {
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo),
@@ -534,15 +554,7 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error {
cParts := []types.CompletedPart{}
for _, el := range parts {
cParts = append(cParts, types.CompletedPart{
ETag: el.ETag,
PartNumber: el.PartNumber,
ChecksumCRC32: el.ChecksumCRC32C,
ChecksumCRC32C: el.ChecksumCRC32C,
ChecksumSHA1: el.ChecksumSHA1,
ChecksumSHA256: el.ChecksumSHA256,
ChecksumCRC64NVME: el.ChecksumCRC64NVME,
})
cParts = append(cParts, completedPartFromPart(el))
}
mpInput := &s3.CompleteMultipartUploadInput{
@@ -566,6 +578,16 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error {
mpInput.ChecksumSHA256 = getPtr("invalid_sha256")
case types.ChecksumAlgorithmCrc64nvme:
mpInput.ChecksumCRC64NVME = getPtr("invalid_crc64nvme")
case types.ChecksumAlgorithmSha512:
mpInput.ChecksumSHA512 = getPtr("invalid_sha512")
case types.ChecksumAlgorithmMd5:
mpInput.ChecksumMD5 = getPtr("invalid_md5")
case types.ChecksumAlgorithmXxhash64:
mpInput.ChecksumXXHASH64 = getPtr("invalid_xxhash64")
case types.ChecksumAlgorithmXxhash3:
mpInput.ChecksumXXHASH3 = getPtr("invalid_xxhash3")
case types.ChecksumAlgorithmXxhash128:
mpInput.ChecksumXXHASH128 = getPtr("invalid_xxhash128")
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
@@ -608,6 +630,26 @@ func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error {
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo),
withChecksumType(el.t))
@@ -623,34 +665,23 @@ func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error {
cParts := []types.CompletedPart{}
for _, el := range parts {
cParts = append(cParts, types.CompletedPart{
ETag: el.ETag,
PartNumber: el.PartNumber,
ChecksumCRC32: el.ChecksumCRC32,
ChecksumCRC32C: el.ChecksumCRC32C,
ChecksumSHA1: el.ChecksumSHA1,
ChecksumSHA256: el.ChecksumSHA256,
ChecksumCRC64NVME: el.ChecksumCRC64NVME,
})
cParts = append(cParts, completedPartFromPart(el))
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
input := &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: cParts,
},
// Provide one of the parts checksum. In any case
// the final checksum will differ from one of the parts checksum
ChecksumCRC32: cParts[0].ChecksumCRC32,
ChecksumCRC32C: cParts[0].ChecksumCRC32C,
ChecksumSHA1: cParts[0].ChecksumSHA1,
ChecksumSHA256: cParts[0].ChecksumSHA256,
ChecksumCRC64NVME: cParts[0].ChecksumCRC64NVME,
ChecksumType: el.t,
})
ChecksumType: el.t,
}
// Provide one of the part checksums. The final checksum will differ
// from a single part checksum.
setCompleteMultipartUploadChecksum(input, el.algo, getCompletedPartChecksum(cParts[0], el.algo))
_, err = s3client.CompleteMultipartUpload(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
return err
@@ -848,6 +879,11 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
types.ChecksumAlgorithmCrc32c,
types.ChecksumAlgorithmSha1,
types.ChecksumAlgorithmSha256,
types.ChecksumAlgorithmSha512,
types.ChecksumAlgorithmMd5,
types.ChecksumAlgorithmXxhash64,
types.ChecksumAlgorithmXxhash3,
types.ChecksumAlgorithmXxhash128,
} {
mp, err := createMp(s3client, bucket, obj, withChecksumType(types.ChecksumTypeComposite), withChecksum(algo))
if err != nil {
@@ -867,29 +903,13 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
completeParts := make([]types.CompletedPart, 0, len(parts))
for _, part := range parts {
switch algo {
case types.ChecksumAlgorithmCrc32:
err = processCompositeChecksum(hasher, getString(part.ChecksumCRC32))
case types.ChecksumAlgorithmCrc32c:
err = processCompositeChecksum(hasher, getString(part.ChecksumCRC32C))
case types.ChecksumAlgorithmSha1:
err = processCompositeChecksum(hasher, getString(part.ChecksumSHA1))
case types.ChecksumAlgorithmSha256:
err = processCompositeChecksum(hasher, getString(part.ChecksumSHA256))
}
err = processCompositeChecksum(hasher, getString(getPartChecksum(part, algo)))
if err != nil {
return fmt.Errorf("test %v failed: %s", i, err)
}
completeParts = append(completeParts, types.CompletedPart{
ETag: part.ETag,
PartNumber: part.PartNumber,
ChecksumCRC32: part.ChecksumCRC32,
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
})
completeParts = append(completeParts, completedPartFromPart(part))
}
checksum := fmt.Sprintf("%s-%v", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), len(parts))
@@ -903,16 +923,7 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
UploadId: mp.UploadId,
}
switch algo {
case types.ChecksumAlgorithmCrc32:
completeMpInput.ChecksumCRC32 = &checksum
case types.ChecksumAlgorithmCrc32c:
completeMpInput.ChecksumCRC32C = &checksum
case types.ChecksumAlgorithmSha1:
completeMpInput.ChecksumSHA1 = &checksum
case types.ChecksumAlgorithmSha256:
completeMpInput.ChecksumSHA256 = &checksum
}
setCompleteMultipartUploadChecksum(completeMpInput, algo, &checksum)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.CompleteMultipartUpload(ctx, completeMpInput)
@@ -921,17 +932,7 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
return fmt.Errorf("test %v failed: %s", i, err)
}
var gotSum string
switch algo {
case types.ChecksumAlgorithmCrc32:
gotSum = getString(res.ChecksumCRC32)
case types.ChecksumAlgorithmCrc32c:
gotSum = getString(res.ChecksumCRC32C)
case types.ChecksumAlgorithmSha1:
gotSum = getString(res.ChecksumSHA1)
case types.ChecksumAlgorithmSha256:
gotSum = getString(res.ChecksumSHA256)
}
gotSum := getString(getCompleteMultipartUploadChecksum(res, algo))
if gotSum != checksum {
return fmt.Errorf("test %v failed: expected the final checksum to be %s, instead got %s", i, checksum, gotSum)
@@ -947,20 +948,27 @@ func CompleteMultipartUpload_invalid_final_composite_checksum(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for i, test := range []struct {
algo types.ChecksumAlgorithm
crc32 *string
crc32c *string
sha1 *string
sha256 *string
algo types.ChecksumAlgorithm
checksum *string
}{
{types.ChecksumAlgorithmCrc32, getPtr("invalid_checksum"), nil, nil, nil},
{types.ChecksumAlgorithmCrc32, getPtr("ImIEBA==-smth"), nil, nil, nil},
{types.ChecksumAlgorithmCrc32c, nil, getPtr("invalid_checksum"), nil, nil},
{types.ChecksumAlgorithmCrc32c, nil, getPtr("AQIDBA==-12a"), nil, nil},
{types.ChecksumAlgorithmSha1, nil, nil, getPtr("invalid_checksum"), nil},
{types.ChecksumAlgorithmSha1, nil, nil, getPtr("2jmj7l5rSw0yVb/vlWAYkK/YBwk=-10-20"), nil},
{types.ChecksumAlgorithmSha256, nil, nil, nil, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha256, nil, nil, nil, getPtr("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=--3")},
{types.ChecksumAlgorithmCrc32, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmCrc32, getPtr("ImIEBA==-smth")},
{types.ChecksumAlgorithmCrc32c, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmCrc32c, getPtr("AQIDBA==-12a")},
{types.ChecksumAlgorithmSha1, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha1, getPtr("2jmj7l5rSw0yVb/vlWAYkK/YBwk=-10-20")},
{types.ChecksumAlgorithmSha256, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha256, getPtr("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=--3")},
{types.ChecksumAlgorithmSha512, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha512, getPtr("z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==-bad")},
{types.ChecksumAlgorithmMd5, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmMd5, getPtr("1B2M2Y8AsgTpgAmY7PhCfg==-bad")},
{types.ChecksumAlgorithmXxhash64, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmXxhash64, getPtr("70bbN1HY6Zk=-bad")},
{types.ChecksumAlgorithmXxhash3, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmXxhash3, getPtr("LQaABTjTlMI=-bad")},
{types.ChecksumAlgorithmXxhash128, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmXxhash128, getPtr("maoG0wFHmNhgAcMkRo1Jfw==-bad")},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(test.algo), withChecksumType(types.ChecksumTypeComposite))
if err != nil {
@@ -975,29 +983,20 @@ func CompleteMultipartUpload_invalid_final_composite_checksum(s *S3Conf) error {
completeParts := make([]types.CompletedPart, 0, len(parts))
for _, part := range parts {
completeParts = append(completeParts, types.CompletedPart{
ETag: part.ETag,
PartNumber: part.PartNumber,
ChecksumCRC32: part.ChecksumCRC32,
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
})
completeParts = append(completeParts, completedPartFromPart(part))
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
input := &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: completeParts,
},
ChecksumCRC32: test.crc32,
ChecksumCRC32C: test.crc32c,
ChecksumSHA1: test.sha1,
ChecksumSHA256: test.sha256,
})
}
setCompleteMultipartUploadChecksum(input, test.algo, test.checksum)
_, err = s3client.CompleteMultipartUpload(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(test.algo))))); err != nil {
return fmt.Errorf("test %v failed: %w", i, err)
@@ -1090,21 +1089,20 @@ func CompleteMultipartUpload_should_ignore_the_final_checksum(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, insted got %v",
*res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, insted got %v",
*res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, insted got %v",
*res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, insted got %v",
*res.ChecksumSHA256)
for _, algo := range []types.ChecksumAlgorithm{
types.ChecksumAlgorithmCrc32,
types.ChecksumAlgorithmCrc32c,
types.ChecksumAlgorithmSha1,
types.ChecksumAlgorithmSha256,
types.ChecksumAlgorithmSha512,
types.ChecksumAlgorithmMd5,
types.ChecksumAlgorithmXxhash64,
types.ChecksumAlgorithmXxhash3,
types.ChecksumAlgorithmXxhash128,
} {
if checksum := getCompleteMultipartUploadChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, insted got %v", algo, *checksum)
}
}
// If no checksum is specified on mp creation, it should default
// to crc64nvme
tests/integration/CopyObject.go
+14 -22
View File
@@ -1461,17 +1461,13 @@ func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error {
if out.CopyObjectResult.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum")
}
if out.CopyObjectResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected empty crc32c checksum")
}
if out.CopyObjectResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected empty sha1 checksum")
}
if out.CopyObjectResult.ChecksumSHA256 != nil {
return fmt.Errorf("expected empty sha256 checksum")
}
if out.CopyObjectResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected empty crc64nvme checksum")
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if algo == types.ChecksumAlgorithmCrc32 {
continue
}
if checksum := getCopyObjectChecksum(out.CopyObjectResult, algo); checksum != nil {
return fmt.Errorf("expected empty %s checksum", algo)
}
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
@@ -1488,17 +1484,13 @@ func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error {
if res.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum")
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected empty crc32c checksum")
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected empty sha1 checksum")
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected empty sha256 checksum")
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected empty crc64nvme checksum")
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if algo == types.ChecksumAlgorithmCrc32 {
continue
}
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected empty %s checksum", algo)
}
}
return nil
tests/integration/CreateMultipartUpload.go
+10
View File
@@ -390,6 +390,11 @@ func CreateMultipartUpload_type_algo_mismatch(s *S3Conf) error {
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc64nvme},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha256},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha512},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmMd5},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash64},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash3},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash128},
} {
_, err := createMp(s3client, bucket, "my-obj", withChecksum(test.algo), withChecksumType(test.chType))
if err := checkApiErr(err, s3err.GetChecksumSchemaMismatchErr(test.algo, test.chType)); err != nil {
@@ -414,6 +419,11 @@ func CreateMultipartUpload_valid_algo_type(s *S3Conf) error {
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha512},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmMd5},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash64},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash3},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash128},
// full object type
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc64nvme},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc32},
tests/integration/GetObject.go
+45 -131
View File
@@ -233,38 +233,13 @@ func GetObject_invalid_parent(s *S3Conf) error {
func GetObject_checksums(s *S3Conf) error {
testName := "GetObject_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -272,7 +247,7 @@ func GetObject_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -281,28 +256,11 @@ func GetObject_checksums(s *S3Conf) error {
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v",
el.key, types.ChecksumTypeFullObject, res.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v",
key, types.ChecksumTypeFullObject, res.ChecksumType)
}
if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32))
}
if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C))
}
if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1))
}
if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256))
}
if getString(res.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC64NVME), getString(res.ChecksumCRC64NVME))
if got, expected := getString(getGetObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got)
}
}
@@ -313,43 +271,31 @@ func GetObject_checksums(s *S3Conf) error {
func GetObject_dir_object_checksum(s *S3Conf) error {
testName := "GetObject_dir_object_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
for i, obj := range []struct {
key string
expectedSum string
for i, test := range []struct {
checksumAlgo types.ChecksumAlgorithm
expectedSum string
}{
{
key: "obj-1/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3/",
expectedSum: "AAAAAAAAAAA=",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
{
key: "obj-4/",
expectedSum: "2jmj7l5rSw0yVb/vlWAYkK/YBwk=",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-5/",
expectedSum: "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{types.ChecksumAlgorithmCrc32, "AAAAAA=="},
{types.ChecksumAlgorithmCrc32c, "AAAAAA=="},
{types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="},
{types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="},
{types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},
{types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="},
{types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="},
{types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="},
{types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="},
{types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, &s3.PutObjectInput{
key := fmt.Sprintf("obj-%v/", i+1)
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj.key,
ChecksumAlgorithm: obj.checksumAlgo,
})
Key: &key,
ChecksumAlgorithm: test.checksumAlgo,
}
setPutObjectChecksum(input, test.checksumAlgo, &test.expectedSum)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, input)
cancel()
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
@@ -358,7 +304,7 @@ func GetObject_dir_object_checksum(s *S3Conf) error {
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &obj.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -368,25 +314,13 @@ func GetObject_dir_object_checksum(s *S3Conf) error {
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("test %v failed: expected the %v object checksum type to be %v, instaed got %v",
i+1, obj.key, types.ChecksumTypeFullObject, res.ChecksumType)
i+1, key, types.ChecksumTypeFullObject, res.ChecksumType)
}
var gotSum *string
switch obj.checksumAlgo {
case types.ChecksumAlgorithmCrc32:
gotSum = res.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
gotSum = res.ChecksumCRC32C
case types.ChecksumAlgorithmCrc64nvme:
gotSum = res.ChecksumCRC64NVME
case types.ChecksumAlgorithmSha1:
gotSum = res.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
gotSum = res.ChecksumSHA256
}
gotSum := getGetObjectChecksum(res, test.checksumAlgo)
if getString(gotSum) != obj.expectedSum {
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, obj.checksumAlgo, obj.expectedSum, getString(gotSum))
if getString(gotSum) != test.expectedSum {
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, test.checksumAlgo, test.expectedSum, getString(gotSum))
}
}
@@ -885,20 +819,10 @@ func GetObject_not_enabled_checksum_mode(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getGetObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -1704,20 +1628,10 @@ func GetObject_ranged_with_checksum_mode(s *S3Conf) error {
testName := "GetObject_ranged_with_checksum_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
checkNoChecksums := func(res *s3.GetObjectOutput) error {
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getGetObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
}
tests/integration/GetObjectAttributes.go
+11 -53
View File
@@ -235,38 +235,13 @@ func GetObjectAttributes_existing_object(s *S3Conf) error {
func GetObjectAttributes_checksums(s *S3Conf) error {
testName := "GetObjectAttributes_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -274,7 +249,7 @@ func GetObjectAttributes_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesChecksum,
},
@@ -288,28 +263,11 @@ func GetObjectAttributes_checksums(s *S3Conf) error {
return fmt.Errorf("expected non-nil checksum in the response")
}
if res.Checksum.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v",
el.key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v",
key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType)
}
if getString(res.Checksum.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.Checksum.ChecksumCRC32))
}
if getString(res.Checksum.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32C), getString(res.Checksum.ChecksumCRC32C))
}
if getString(res.Checksum.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA1), getString(res.Checksum.ChecksumSHA1))
}
if getString(res.Checksum.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA256), getString(res.Checksum.ChecksumSHA256))
}
if getString(res.Checksum.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC64NVME), getString(res.Checksum.ChecksumCRC64NVME))
if got, expected := getString(getObjectAttributesChecksum(res.Checksum, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %s, instead got %s", checksumAlgo, expected, got)
}
}
return nil
tests/integration/HeadObject.go
+18 -72
View File
@@ -150,17 +150,10 @@ func HeadObject_not_enabled_checksum_mode(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -170,38 +163,13 @@ func HeadObject_not_enabled_checksum_mode(s *S3Conf) error {
func HeadObject_checksums(s *S3Conf) error {
testName := "HeadObject_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*200), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*200), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -209,7 +177,7 @@ func HeadObject_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -218,22 +186,10 @@ func HeadObject_checksums(s *S3Conf) error {
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v", key, types.ChecksumTypeFullObject, res.ChecksumType)
}
if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32))
}
if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C))
}
if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1))
}
if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256))
}
if getString(res.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v", getString(out.res.ChecksumCRC64NVME), getString(res.ChecksumCRC64NVME))
if got, expected := getString(getHeadObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got)
}
}
@@ -1311,20 +1267,10 @@ func HeadObject_ranged_with_checksum_mode(s *S3Conf) error {
testName := "HeadObject_ranged_with_checksum_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
checkNoChecksums := func(res *s3.HeadObjectOutput) error {
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
}
tests/integration/ListMultipartUploads.go
+26
View File
@@ -612,6 +612,32 @@ func ListMultipartUploads_with_checksums(s *S3Conf) error {
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
{
obj: "obj-6",
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
{
obj: "obj-7",
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
{
obj: "obj-8",
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
{
obj: "obj-9",
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
// obj-91 to preserve the lexicographical order
{
obj: "obj-91",
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
} {
key := el.obj
mp, err := createMp(s3client, bucket, key, withChecksum(el.algo), withChecksumType(el.t))
tests/integration/ListObjects.go
+1 -1
View File
@@ -349,7 +349,7 @@ func ListObjects_with_checksum(s *S3Conf) error {
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: el,
}, s3client)
}, s3client, withPutObjectChecksumAlgo(el))
if err != nil {
return err
}
tests/integration/ListObjectsV2.go
+1 -1
View File
@@ -578,7 +578,7 @@ func ListObjectsV2_with_checksum(s *S3Conf) error {
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: el,
}, s3client)
}, s3client, withPutObjectChecksumAlgo(el))
if err != nil {
return err
}
tests/integration/PostObject.go
+5
View File
@@ -1070,6 +1070,11 @@ func PostObject_checksums_success(s *S3Conf) error {
{"x-amz-checksum-crc64nvme", "rsrzr5yYqFU="},
{"x-amz-checksum-sha1", "9I3YU4IIYIFsddVND1hNyGMyenw="},
{"x-amz-checksum-sha256", "kW8AJ6V1B0znKjMXd8NHjWUT94alkb2JLaGld78jNfk="},
{"x-amz-checksum-sha512", "Dh4h7PEF7IU9JNcohnrXBhPCFmOkaTB0sqNhnBvTnWa1iMM3I7tGbHJCToDjymPCSQeKs0e6uUKFAOfuQwWdDQ=="},
{"x-amz-checksum-md5", "63M6AMDJ0zbmVpGjerVCkw=="},
{"x-amz-checksum-xxhash64", "+lb36/ER8bo="},
{"x-amz-checksum-xxhash3", "jw+pSh/pbMQ="},
{"x-amz-checksum-xxhash128", "8BLDqqIWji+ITOsp/JjN/Q=="},
} {
resp, err := sendPostObject(PostRequestConfig{
bucket: bucket,
tests/integration/PutObject.go
+58 -185
View File
@@ -702,98 +702,31 @@ func PutObject_invalid_checksum_header(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for i, el := range []struct {
algo string
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
// CRC32 tests
{
algo: "crc32",
crc32: getPtr(""),
},
{
algo: "crc32",
crc32: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32",
crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32
},
// CRC32C tests
{
algo: "crc32c",
crc32c: getPtr(""),
},
{
algo: "crc32c",
crc32c: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32c",
crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c
},
// SHA1 tests
{
algo: "sha1",
sha1: getPtr(""),
},
{
algo: "sha1",
sha1: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha1",
sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1
},
// SHA256 tests
{
algo: "sha256",
sha256: getPtr(""),
},
{
algo: "sha256",
sha256: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha256",
sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56
},
// CRC64Nvme tests
{
algo: "crc64nvme",
sha256: getPtr(""),
},
{
algo: "crc64nvme",
sha256: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc64nvme",
sha256: getPtr("ZHNhZmRzYWZzZGFmZHNhZg=="), // valid base64 but not crc64nvme
},
} {
_, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
}, s3client)
i := 0
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
// tests against:
// - empty string
// - invalid base64
// - valid base64, but invalid checksum
for _, checksum := range []string{"", "invalid_base64!", "c2RhZnNhZGZzZGFm"} {
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}
setPutObjectChecksum(input, algo, getPtr(checksum))
// FIXME: The error message for PutObject is not properly serialized by the sdk
// References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921
_, err := putObjectWithData(int64((i+1)*100), input, s3client)
i++
// if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil {
// return err
// }
if err := checkSdkApiErr(err, "InvalidRequest"); err != nil {
return err
// FIXME: The error message for PutObject is not properly serialized by the sdk
// References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921
// if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil {
// return err
// }
if err := checkSdkApiErr(err, "InvalidRequest"); err != nil {
return err
}
}
}
@@ -807,62 +740,33 @@ func PutObject_incorrect_checksums(s *S3Conf) error {
obj := "my-obj"
dirObj := "dir-object/"
for i, el := range []struct {
algo types.ChecksumAlgorithm
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
{
algo: types.ChecksumAlgorithmCrc32,
crc32: getPtr("DUoRhQ=="),
},
{
algo: types.ChecksumAlgorithmCrc32c,
crc32c: getPtr("yZRlqg=="),
},
{
algo: types.ChecksumAlgorithmSha1,
sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="),
},
{
algo: types.ChecksumAlgorithmSha256,
sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="),
},
{
algo: types.ChecksumAlgorithmCrc64nvme,
crc64nvme: getPtr("sV264W+gYBI="),
},
} {
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
wrongChecksum, err := wrongChecksumForAlgorithm(algo)
if err != nil {
return err
}
// test for file object
_, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
}, s3client)
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}
setPutObjectChecksum(input, algo, &wrongChecksum)
_, err = putObjectWithData(int64((i+1)*100), input, s3client)
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil {
return err
}
// test for directory object
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &bucket,
Key: &dirObj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
})
input = &s3.PutObjectInput{
Bucket: &bucket,
Key: &dirObj,
}
setPutObjectChecksum(input, algo, &wrongChecksum)
_, err = s3client.PutObject(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil {
return err
}
}
@@ -952,13 +856,20 @@ func PutObject_dir_object_checksums_success(s *S3Conf) error {
{types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="},
{types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="},
{types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},
{types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="},
{types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="},
{types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="},
{types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="},
{types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.PutObject(ctx, &s3.PutObjectInput{
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: getPtr(fmt.Sprintf("obj-%v/", i)),
ChecksumAlgorithm: test.algo,
})
}
setPutObjectChecksum(input, test.algo, &test.checksumValue)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.PutObject(ctx, input)
cancel()
if err != nil {
return err
@@ -968,27 +879,8 @@ func PutObject_dir_object_checksums_success(s *S3Conf) error {
return fmt.Errorf("expected the checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType)
}
switch test.algo {
case types.ChecksumAlgorithmCrc32:
if getString(res.ChecksumCRC32) != test.checksumValue {
return fmt.Errorf("expected the crc32 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32))
}
case types.ChecksumAlgorithmCrc32c:
if getString(res.ChecksumCRC32C) != test.checksumValue {
return fmt.Errorf("expected the crc32c checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32C))
}
case types.ChecksumAlgorithmSha1:
if getString(res.ChecksumSHA1) != test.checksumValue {
return fmt.Errorf("expected the sha1 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA1))
}
case types.ChecksumAlgorithmSha256:
if getString(res.ChecksumSHA256) != test.checksumValue {
return fmt.Errorf("expected the sha256 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA256))
}
case types.ChecksumAlgorithmCrc64nvme:
if getString(res.ChecksumCRC64NVME) != test.checksumValue {
return fmt.Errorf("expected the crc64nvme checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC64NVME))
}
if got := getString(getPutObjectChecksum(res, test.algo)); got != test.checksumValue {
return fmt.Errorf("expected the %s checksum value to be %s, instead got %s", test.algo, test.checksumValue, got)
}
}
@@ -1006,7 +898,7 @@ func PutObject_checksums_success(s *S3Conf) error {
Bucket: &bucket,
Key: &obj,
ChecksumAlgorithm: algo,
}, s3client)
}, s3client, withPutObjectChecksumAlgo(algo))
if err != nil {
return err
}
@@ -1015,27 +907,8 @@ func PutObject_checksums_success(s *S3Conf) error {
return fmt.Errorf("expected the object checksum type to be %v, instead got %v", types.ChecksumTypeFullObject, res.res.ChecksumType)
}
switch algo {
case types.ChecksumAlgorithmCrc32:
if res.res.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum in the response")
}
case types.ChecksumAlgorithmCrc32c:
if res.res.ChecksumCRC32C == nil {
return fmt.Errorf("expected non empty crc32c checksum in the response")
}
case types.ChecksumAlgorithmSha1:
if res.res.ChecksumSHA1 == nil {
return fmt.Errorf("expected non empty sha1 checksum in the response")
}
case types.ChecksumAlgorithmSha256:
if res.res.ChecksumSHA256 == nil {
return fmt.Errorf("expected non empty sha256 checksum in the response")
}
case types.ChecksumAlgorithmCrc64nvme:
if res.res.ChecksumCRC64NVME == nil {
return fmt.Errorf("expected non empty crc64nvme checksum in the response")
}
if getPutObjectChecksum(res.res, algo) == nil {
return fmt.Errorf("expected non empty %s checksum in the response", algo)
}
}
tests/integration/UploadPart.go
+51 -213
View File
@@ -155,95 +155,26 @@ func UploadPart_invalid_checksum_header(s *S3Conf) error {
partNumber := int32(1)
for _, el := range []struct {
algo string
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
// CRC32 tests
{
algo: "crc32",
crc32: getPtr(""),
},
{
algo: "crc32",
crc32: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32",
crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32
},
// CRC32C tests
{
algo: "crc32c",
crc32c: getPtr(""),
},
{
algo: "crc32c",
crc32c: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32c",
crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c
},
// SHA1 tests
{
algo: "sha1",
sha1: getPtr(""),
},
{
algo: "sha1",
sha1: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha1",
sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1
},
// SHA256 tests
{
algo: "sha256",
sha256: getPtr(""),
},
{
algo: "sha256",
sha256: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha256",
sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56
},
// CRC64NVME tests
{
algo: "crc64nvme",
crc64nvme: getPtr(""),
},
{
algo: "crc64nvme",
crc64nvme: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc64nvme",
crc64nvme: getPtr("ZHNhZmRzYWZzZGFmZHNhZg=="), // valid base64 but not crc64nvme
},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
PartNumber: &partNumber,
UploadId: mp.UploadId,
})
cancel()
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", el.algo))); err != nil {
return err
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
// tests against:
// - empty string
// - invalid base64
// - valid base64, but invalid checksum
for _, checksum := range []string{"", "invalid_base64!", "c2RhZnNhZGZzZGFm"} {
input := &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
PartNumber: &partNumber,
UploadId: mp.UploadId,
}
setUploadPartChecksum(input, algo, getPtr(checksum))
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.UploadPart(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(checksumHeaderName(algo))); err != nil {
return err
}
}
}
@@ -336,36 +267,13 @@ func UploadPart_incorrect_checksums(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for _, el := range []struct {
algo types.ChecksumAlgorithm
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
{
algo: types.ChecksumAlgorithmCrc32,
crc32: getPtr("DUoRhQ=="),
},
{
algo: types.ChecksumAlgorithmCrc32c,
crc32c: getPtr("yZRlqg=="),
},
{
algo: types.ChecksumAlgorithmSha1,
sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="),
},
{
algo: types.ChecksumAlgorithmSha256,
sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="),
},
{
algo: types.ChecksumAlgorithmCrc64nvme,
crc64nvme: getPtr("MN2ofvMjpIQ="),
},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo))
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
wrongChecksum, err := wrongChecksumForAlgorithm(algo)
if err != nil {
return err
}
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
@@ -374,20 +282,17 @@ func UploadPart_incorrect_checksums(s *S3Conf) error {
partNumber := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
UploadId: mp.UploadId,
PartNumber: &partNumber,
Body: body,
})
input := &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
Body: body,
}
setUploadPartChecksum(input, algo, &wrongChecksum)
_, err = s3client.UploadPart(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil {
return err
}
}
@@ -479,6 +384,11 @@ func UploadPart_no_checksum_with_composite_checksum_type(s *S3Conf) error {
types.ChecksumAlgorithmCrc32c,
types.ChecksumAlgorithmSha1,
types.ChecksumAlgorithmSha256,
types.ChecksumAlgorithmSha512,
types.ChecksumAlgorithmMd5,
types.ChecksumAlgorithmXxhash64,
types.ChecksumAlgorithmXxhash3,
types.ChecksumAlgorithmXxhash128,
} {
mp, err := createMp(customClient, bucket, obj, withChecksum(algo), withChecksumType(types.ChecksumTypeComposite))
if err != nil {
@@ -493,8 +403,8 @@ func UploadPart_no_checksum_with_composite_checksum_type(s *S3Conf) error {
})
}
func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Conf) error {
testName := "UploadPart_should_calculate_checksum_if_only_algorithm_is_provided"
func UploadPart_with_checksums_success(s *S3Conf) error {
testName := "UploadPart_with_checksums_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
customClient := s3.NewFromConfig(s.Config(), func(o *s3.Options) {
o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset
@@ -512,6 +422,11 @@ func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Con
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha512},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmMd5},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash64},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash3},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash128},
} {
mp, err := createMp(customClient, bucket, obj, withChecksum(test.chAlgo), withChecksumType(test.chType))
if err != nil {
@@ -527,91 +442,14 @@ func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Con
return fmt.Errorf("expected 1 uploaded part, instaed got %d", len(parts))
}
part := parts[0]
switch test.chAlgo {
case types.ChecksumAlgorithmCrc32:
if getString(part.ChecksumCRC32) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32))
}
case types.ChecksumAlgorithmCrc32c:
if getString(part.ChecksumCRC32C) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32C))
}
case types.ChecksumAlgorithmCrc64nvme:
if getString(part.ChecksumCRC64NVME) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC64NVME))
}
case types.ChecksumAlgorithmSha1:
if getString(part.ChecksumSHA1) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA1))
}
case types.ChecksumAlgorithmSha256:
if getString(part.ChecksumSHA256) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA256))
}
if got := getString(getPartChecksum(parts[0], test.chAlgo)); got != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, got)
}
}
return nil
})
}
func UploadPart_with_checksums_success(s *S3Conf) error {
testName := "UploadPart_with_checksums_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
partNumber := int32(1)
data := make([]byte, i*100)
rand.Read(data)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
ChecksumAlgorithm: algo,
UploadId: mp.UploadId,
PartNumber: &partNumber,
Body: bytes.NewReader(data),
})
cancel()
if err != nil {
return err
}
switch algo {
case types.ChecksumAlgorithmCrc32:
if res.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum in the response")
}
case types.ChecksumAlgorithmCrc32c:
if res.ChecksumCRC32C == nil {
return fmt.Errorf("expected non empty crc32c checksum in the response")
}
case types.ChecksumAlgorithmSha1:
if res.ChecksumSHA1 == nil {
return fmt.Errorf("expected non empty sha1 checksum in the response")
}
case types.ChecksumAlgorithmSha256:
if res.ChecksumSHA256 == nil {
return fmt.Errorf("expected non empty sha256 checksum in the response")
}
case types.ChecksumAlgorithmCrc64nvme:
if res.ChecksumCRC64NVME == nil {
return fmt.Errorf("expected non empty crc64nvme checksum in the response")
}
}
}
return nil
})
}
func UploadPart_non_existing_key(s *S3Conf) error {
testName := "UploadPart_non_existing_key"
partNumber := int32(1)
tests/integration/UploadPartCopy.go
+83 -110
View File
@@ -845,57 +845,50 @@ func UploadPartCopy_conditional_reads(s *S3Conf) error {
func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error {
testName := "UploadPartCopy_should_copy_the_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
srcObj := "source-object"
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
obj := fmt.Sprintf("my-obj-%v", i+1)
srcObj := fmt.Sprintf("source-object-%v", i+1)
mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32))
if err != nil {
return err
}
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
out, err := putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: types.ChecksumAlgorithmCrc32,
}, s3client)
if err != nil {
return err
}
out, err := putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: algo,
}, s3client, withPutObjectChecksumAlgo(algo))
if err != nil {
return err
}
partNumber := int32(1)
partNumber := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
if getString(res.CopyPartResult.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.CopyPartResult.ChecksumCRC32))
}
if res.CopyPartResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32C)
}
if res.CopyPartResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA1)
}
if res.CopyPartResult.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA256)
}
if res.CopyPartResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC64NVME)
if got, expected := getString(getUploadPartCopyChecksum(res.CopyPartResult, algo)), getString(getPutObjectChecksum(out.res, algo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", algo, expected, got)
}
for _, otherAlgo := range types.ChecksumAlgorithmCrc32.Values() {
if otherAlgo == algo {
continue
}
if checksum := getUploadPartCopyChecksum(res.CopyPartResult, otherAlgo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", otherAlgo, *checksum)
}
}
}
return nil
@@ -937,25 +930,10 @@ func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error {
return err
}
if res.CopyPartResult.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32)
}
if res.CopyPartResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32C)
}
if res.CopyPartResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA1)
}
if res.CopyPartResult.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA256)
}
if res.CopyPartResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getUploadPartCopyChecksum(res.CopyPartResult, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -965,56 +943,51 @@ func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error {
func UploadPartCopy_should_calculate_the_checksum(s *S3Conf) error {
testName := "UploadPartCopy_should_calculate_the_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
srcObj := "source-object"
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
obj := fmt.Sprintf("my-obj-%v", i+1)
srcObj := fmt.Sprintf("source-object-%v", i+1)
mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha256))
if err != nil {
return err
}
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
_, err = putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: types.ChecksumAlgorithmSha1, // different from the mp checksum (sha256)
}, s3client)
if err != nil {
return err
}
_, err = putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: types.ChecksumAlgorithmSha1,
}, s3client)
if err != nil {
return err
}
partNumber := int32(1)
partNumber := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
if res.CopyPartResult.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32)
}
if res.CopyPartResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32C)
}
if res.CopyPartResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC64NVME)
}
if res.CopyPartResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA1)
}
if getString(res.CopyPartResult.ChecksumSHA256) == "" {
return fmt.Errorf("expected non empty sha256 checksum")
for _, otherAlgo := range types.ChecksumAlgorithmCrc32.Values() {
checksum := getUploadPartCopyChecksum(res.CopyPartResult, otherAlgo)
if otherAlgo == algo {
if getString(checksum) == "" {
return fmt.Errorf("expected non empty %s checksum", algo)
}
continue
}
if checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", otherAlgo, *checksum)
}
}
}
return nil
tests/integration/group-tests.go
+1 -3
View File
@@ -445,7 +445,6 @@ func TestUploadPart(ts *TestState) {
ts.Run(UploadPart_incorrect_checksums)
ts.Run(UploadPart_no_checksum_with_full_object_checksum_type)
ts.Run(UploadPart_no_checksum_with_composite_checksum_type)
ts.Run(UploadPart_should_calculate_checksum_if_only_algorithm_is_provided)
ts.Run(UploadPart_with_checksums_success)
}
ts.Run(UploadPart_success)
@@ -552,7 +551,7 @@ func TestCompleteMultipartUpload(ts *TestState) {
}
ts.Run(CompleteMultipartUpload_success)
ts.Run(CompleteMultipartUpload_already_completed)
if !ts.conf.azureTests {
if !(ts.conf.azureTests || ts.conf.sidecarTests) {
ts.Run(CompleteMultipartUpload_racey_success)
ts.Run(CompleteMultipartUpload_racey_data_integrity)
}
@@ -1589,7 +1588,6 @@ func GetIntTests() IntTests {
"UploadPart_incorrect_checksums": UploadPart_incorrect_checksums,
"UploadPart_no_checksum_with_full_object_checksum_type": UploadPart_no_checksum_with_full_object_checksum_type,
"UploadPart_no_checksum_with_composite_checksum_type": UploadPart_no_checksum_with_composite_checksum_type,
"UploadPart_should_calculate_checksum_if_only_algorithm_is_provided": UploadPart_should_calculate_checksum_if_only_algorithm_is_provided,
"UploadPart_with_checksums_success": UploadPart_with_checksums_success,
"UploadPart_success": UploadPart_success,
"UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket,
tests/integration/s3conf.go
+4
View File
@@ -43,6 +43,7 @@ type S3Conf struct {
debug bool
versioningEnabled bool
azureTests bool
sidecarTests bool
tlsStatus bool
httpClient *http.Client
}
@@ -105,6 +106,9 @@ func WithVersioningEnabled() Option {
func WithAzureMode() Option {
return func(s *S3Conf) { s.azureTests = true }
}
func WithSidecarMode() Option {
return func(s *S3Conf) { s.sidecarTests = true }
}
func WithTLSStatus(ts bool) Option {
return func(s *S3Conf) { s.tlsStatus = ts }
}
tests/integration/signed_streaming_payload_trailer.go
+27 -23
View File
@@ -108,24 +108,17 @@ func SignedStreamingPayloadTrailer_invalid_checksum(s *S3Conf) error {
testName := "SignedStreamingPayloadTrailer_invalid_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
trailerHdr string
trailingChecksum string
}{
{"x-amz-checksum-crc32", "x-amz-checksum-crc32:invalid"},
{"x-amz-checksum-crc32c", "x-amz-checksum-crc32c:invalid"},
{"x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:invalid"},
{"x-amz-checksum-sha1", "x-amz-checksum-sha1:invalid"},
{"x-amz-checksum-sha256", "x-amz-checksum-sha256:invalid"},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": test.trailerHdr,
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
trailerHdr := checksumHeaderName(algo)
trailingChecksum := fmt.Sprintf("%s:invalid", trailerHdr)
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": trailerHdr,
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(test.trailerHdr), apiErr); err != nil {
if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(trailerHdr), apiErr); err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
}
@@ -139,18 +132,24 @@ func SignedStreamingPayloadTrailer_bad_digest(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
algo types.ChecksumAlgorithm
trailerHdr string
trailingChecksum string
algo types.ChecksumAlgorithm
checksum string
}{
{types.ChecksumAlgorithmCrc32, "x-amz-checksum-crc32", "x-amz-checksum-crc32:NhCmhg=="},
{types.ChecksumAlgorithmCrc32c, "x-amz-checksum-crc32c", "x-amz-checksum-crc32c:+Cy97w=="},
{types.ChecksumAlgorithmCrc64nvme, "x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:QFRKMGE3tuw="},
{types.ChecksumAlgorithmSha1, "x-amz-checksum-sha1", "x-amz-checksum-sha1:qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{types.ChecksumAlgorithmSha256, "x-amz-checksum-sha256", "x-amz-checksum-sha256:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
{types.ChecksumAlgorithmCrc32, "NhCmhg=="},
{types.ChecksumAlgorithmCrc32c, "+Cy97w=="},
{types.ChecksumAlgorithmCrc64nvme, "QFRKMGE3tuw="},
{types.ChecksumAlgorithmSha1, "qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{types.ChecksumAlgorithmSha256, "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
{types.ChecksumAlgorithmSha512, "mD1D3f9tqQ9qXTthckRqH/4ii4A/5k/dXc+rVkYHioloUf6C9iPJ1uVlSz0vNjoE7BfPtitgdDepx8Ey1RHlIg=="},
{types.ChecksumAlgorithmMd5, "Q3uTDbhLgHnC3YBKcZNrXw=="},
{types.ChecksumAlgorithmXxhash64, "dVz0EuroMyA="},
{types.ChecksumAlgorithmXxhash3, "OHmyyhTeg1E="},
{types.ChecksumAlgorithmXxhash128, "6fhA2KZsN830zx0e5P183w=="},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": test.trailerHdr,
trailerHdr := checksumHeaderName(test.algo)
trailingChecksum := fmt.Sprintf("%s:%s", trailerHdr, test.checksum)
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": trailerHdr,
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
@@ -178,6 +177,11 @@ func SignedStreamingPayloadTrailer_success(s *S3Conf) error {
{"x-amz-checksum-crc64nvme", "dYnI3/Fh0gM="},
{"x-amz-checksum-sha1", "8O8FwCfmd5fCbCBvH09mrKMVoHU="},
{"x-amz-checksum-sha256", "OoSow5X4zTIPl27MtdFdYT+9O3C367C75+Cb2MFtRBc="},
{"x-amz-checksum-sha512", "uiz+VuENyLjzFxWrdmbN/NdIdldj/V3saJF6FsckcYu6xu26fM0CfAaTySaoFJmIdI2m5wbYMJxtShQ1PXk3Tg=="},
{"x-amz-checksum-md5", "EtWtthXtCk3RxUiXKw+ydw=="},
{"x-amz-checksum-xxhash64", "McmxUfNFLUs="},
{"x-amz-checksum-xxhash3", "tLh+aNltTM4="},
{"x-amz-checksum-xxhash128", "MXqrLVobrQkTBf82+7i9AQ=="},
} {
headers, apiErr, err := testSignedStreamingObjectPut(
s,
tests/integration/unsigned_streaming_payload_trailer.go
+30 -9
View File
@@ -134,6 +134,11 @@ func UnsignedStreamingPayloadTrailer_multiple_checksum_headers(s *S3Conf) error
{"crc64nvme", "QFRKMGE3tuw="},
{"sha1", "qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{"sha256", "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
{"sha512", "uiz+VuENyLjzFxWrdmbN/NdIdldj/V3saJF6FsckcYu6xu26fM0CfAaTySaoFJmIdI2m5wbYMJxtShQ1PXk3Tg=="},
{"md5", "EtWtthXtCk3RxUiXKw+ydw=="},
{"xxhash64", "McmxUfNFLUs="},
{"xxhash3", "tLh+aNltTM4="},
{"xxhash128", "MXqrLVobrQkTBf82+7i9AQ=="},
} {
reqHeaders := map[string]string{
"x-amz-decoded-content-length": "5",
@@ -348,6 +353,11 @@ func UnsignedStreamingPayloadTrailer_no_payload_trailer_only_headers(s *S3Conf)
{"crc64nvme", "SmzZ/LTp1CA="},
{"sha1", "L7XhNBn8iSRoZeejJPR27GJOh0A="},
{"sha256", "fRpUEnsiJQL1t5tfsIAwYRUqRPkrN+I8ZSe69mXU2po="},
{"sha512", "1xakGIVptoqxtt+sF45XARTN8Oo6HMDjFIbD5BJBvGp2Qk6MN6sm8Jb8he+YhsjLY0GH9P3f9kX7CZ8f9UxrjA=="},
{"md5", "esZsDxSN6VGbi9JkMSxNZA=="},
{"xxhash64", "GGCUDikCgi0="},
{"xxhash3", "WkDcP9RMBS8="},
{"xxhash128", "Kq/YOGmlnDE/55jA7aptxg=="},
} {
csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key)
reqHeaders := map[string]string{
@@ -387,6 +397,11 @@ func UnsignedStreamingPayloadTrailer_success_both_sdk_algo_and_trailer(s *S3Conf
{"crc64nvme", "SmzZ/LTp1CA="},
{"sha1", "L7XhNBn8iSRoZeejJPR27GJOh0A="},
{"sha256", "fRpUEnsiJQL1t5tfsIAwYRUqRPkrN+I8ZSe69mXU2po="},
{"sha512", "1xakGIVptoqxtt+sF45XARTN8Oo6HMDjFIbD5BJBvGp2Qk6MN6sm8Jb8he+YhsjLY0GH9P3f9kX7CZ8f9UxrjA=="},
{"md5", "esZsDxSN6VGbi9JkMSxNZA=="},
{"xxhash64", "GGCUDikCgi0="},
{"xxhash3", "WkDcP9RMBS8="},
{"xxhash128", "Kq/YOGmlnDE/55jA7aptxg=="},
} {
csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key)
reqHeaders := map[string]string{
@@ -508,23 +523,29 @@ func UnsignedStreamingPayloadTrailer_UploadPart_success_with_trailer(s *S3Conf)
object := "my-object"
for i, test := range []struct {
key string
algo types.ChecksumAlgorithm
value string
}{
{"crc32", "QWaN2w=="},
{"crc32c", "R/I7iQ=="},
{"crc64nvme", "dPVWc2vU1+Q="},
{"sha1", "YR/1TvTYOJz5gtqVFoBJBtmTibY="},
{"sha256", "eXuwq/95jXIAr3aF3KeQHt/8Ur8mUA1b2XKCZY7iQVI="},
{types.ChecksumAlgorithmCrc32, "QWaN2w=="},
{types.ChecksumAlgorithmCrc32c, "R/I7iQ=="},
{types.ChecksumAlgorithmCrc64nvme, "dPVWc2vU1+Q="},
{types.ChecksumAlgorithmSha1, "YR/1TvTYOJz5gtqVFoBJBtmTibY="},
{types.ChecksumAlgorithmSha256, "eXuwq/95jXIAr3aF3KeQHt/8Ur8mUA1b2XKCZY7iQVI="},
{types.ChecksumAlgorithmSha512, "6tQCd6X50h2wXt5deAY0eKF5Xb1LLbSvkZt+Bqczz8bd1+rH+VYZSgWhjOG4zJ41K0kLmDQxuDyOeHMKqKQGNA=="},
{types.ChecksumAlgorithmMd5, "Mb+5cwrlGvc5U7pyDIZg1w=="},
{types.ChecksumAlgorithmXxhash64, "vYOmFpsGXtY="},
{types.ChecksumAlgorithmXxhash3, "9J6xx21X+f8="},
{types.ChecksumAlgorithmXxhash128, "menoCZOCZ2Acv4XMxaUR9w=="},
} {
mp, err := createMp(s3client, bucket, object, withChecksum(types.ChecksumAlgorithm(strings.ToUpper(test.key))))
mp, err := createMp(s3client, bucket, object, withChecksum(test.algo))
if err != nil {
return err
}
csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key)
csumHdr := checksumHeaderName(test.algo)
reqHeaders := map[string]string{
"x-amz-decoded-content-length": "10",
"x-amz-sdk-checksum-algorithm": test.key,
"x-amz-sdk-checksum-algorithm": strings.ToLower(string(test.algo)),
"x-amz-trailer": csumHdr,
}
body := bytes.NewBuffer([]byte("A\r\ndummy data\r\n0\r\n"))
tests/integration/utils.go
+449 -27
View File
@@ -22,6 +22,7 @@ import (
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
@@ -53,7 +54,9 @@ import (
"github.com/aws/smithy-go"
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
)
@@ -547,21 +550,46 @@ func hasPrefixName(prefixes []types.CommonPrefix, names []string) bool {
return true
}
type putObjectCfg struct {
checksumAlgorithm types.ChecksumAlgorithm
}
type putObjectOpt func(*putObjectCfg)
func withPutObjectChecksumAlgo(algo types.ChecksumAlgorithm) putObjectOpt {
return func(poc *putObjectCfg) { poc.checksumAlgorithm = algo }
}
type putObjectOutput struct {
csum [32]byte
data []byte
res *s3.PutObjectOutput
}
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) {
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client, opts ...putObjectOpt) (*putObjectOutput, error) {
cfg := &putObjectCfg{}
for _, opt := range opts {
opt(cfg)
}
var csum [32]byte
var data []byte
if input.Body == nil && lgth != 0 {
data = make([]byte, lgth)
rand.Read(data)
csum = sha256.Sum256(data)
r := bytes.NewReader(data)
input.Body = r
if cfg.checksumAlgorithm != "" {
hasher, err := NewHasher(cfg.checksumAlgorithm)
if err != nil {
return nil, err
}
hasher.Write(data)
sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
setPutObjectChecksum(input, cfg.checksumAlgorithm, &sum)
}
input.Body = bytes.NewReader(data)
}
ctx, cancel := context.WithTimeout(context.Background(), longTimeout)
@@ -713,6 +741,36 @@ func compareParts(parts1, parts2 []types.Part) bool {
return false
}
}
if prt.ChecksumSHA512 != nil {
if *prt.ChecksumSHA512 != getString(parts2[i].ChecksumSHA512) {
fmt.Printf("sha512 checksums are not equal, %v != %v\n", *prt.ChecksumSHA512, getString(parts2[i].ChecksumSHA512))
return false
}
}
if prt.ChecksumMD5 != nil {
if *prt.ChecksumMD5 != getString(parts2[i].ChecksumMD5) {
fmt.Printf("md5 checksums are not equal, %v != %v\n", *prt.ChecksumMD5, getString(parts2[i].ChecksumMD5))
return false
}
}
if prt.ChecksumXXHASH64 != nil {
if *prt.ChecksumXXHASH64 != getString(parts2[i].ChecksumXXHASH64) {
fmt.Printf("xxhash64 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH64, getString(parts2[i].ChecksumXXHASH64))
return false
}
}
if prt.ChecksumXXHASH3 != nil {
if *prt.ChecksumXXHASH3 != getString(parts2[i].ChecksumXXHASH3) {
fmt.Printf("xxhash3 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH3, getString(parts2[i].ChecksumXXHASH3))
return false
}
}
if prt.ChecksumXXHASH128 != nil {
if *prt.ChecksumXXHASH128 != getString(parts2[i].ChecksumXXHASH128) {
fmt.Printf("xxhash128 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH128, getString(parts2[i].ChecksumXXHASH128))
return false
}
}
}
return true
}
@@ -775,6 +833,299 @@ func getPtr[T any](str T) *T {
return &str
}
func checksumHeaderName(algo types.ChecksumAlgorithm) string {
return fmt.Sprintf("x-amz-checksum-%s", strings.ToLower(string(algo)))
}
type checksumFields struct {
CRC32 **string
CRC32C **string
SHA1 **string
SHA256 **string
CRC64NVME **string
SHA512 **string
MD5 **string
XXHASH64 **string
XXHASH3 **string
XXHASH128 **string
}
func selectChecksum(algo types.ChecksumAlgorithm, fields checksumFields) **string {
switch algo {
case types.ChecksumAlgorithmCrc32:
return fields.CRC32
case types.ChecksumAlgorithmCrc32c:
return fields.CRC32C
case types.ChecksumAlgorithmSha1:
return fields.SHA1
case types.ChecksumAlgorithmSha256:
return fields.SHA256
case types.ChecksumAlgorithmCrc64nvme:
return fields.CRC64NVME
case types.ChecksumAlgorithmSha512:
return fields.SHA512
case types.ChecksumAlgorithmMd5:
return fields.MD5
case types.ChecksumAlgorithmXxhash64:
return fields.XXHASH64
case types.ChecksumAlgorithmXxhash3:
return fields.XXHASH3
case types.ChecksumAlgorithmXxhash128:
return fields.XXHASH128
default:
return nil
}
}
func getChecksum(algo types.ChecksumAlgorithm, fields checksumFields) *string {
if checksum := selectChecksum(algo, fields); checksum != nil {
return *checksum
}
return nil
}
func setChecksum(algo types.ChecksumAlgorithm, fields checksumFields, checksum *string) {
if selected := selectChecksum(algo, fields); selected != nil {
*selected = checksum
}
}
func getPartChecksum(part types.Part, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &part.ChecksumCRC32,
CRC32C: &part.ChecksumCRC32C,
SHA1: &part.ChecksumSHA1,
SHA256: &part.ChecksumSHA256,
CRC64NVME: &part.ChecksumCRC64NVME,
SHA512: &part.ChecksumSHA512,
MD5: &part.ChecksumMD5,
XXHASH64: &part.ChecksumXXHASH64,
XXHASH3: &part.ChecksumXXHASH3,
XXHASH128: &part.ChecksumXXHASH128,
})
}
func setPartChecksum(part *types.Part, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &part.ChecksumCRC32,
CRC32C: &part.ChecksumCRC32C,
SHA1: &part.ChecksumSHA1,
SHA256: &part.ChecksumSHA256,
CRC64NVME: &part.ChecksumCRC64NVME,
SHA512: &part.ChecksumSHA512,
MD5: &part.ChecksumMD5,
XXHASH64: &part.ChecksumXXHASH64,
XXHASH3: &part.ChecksumXXHASH3,
XXHASH128: &part.ChecksumXXHASH128,
}, checksum)
}
func getCompletedPartChecksum(part types.CompletedPart, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &part.ChecksumCRC32,
CRC32C: &part.ChecksumCRC32C,
SHA1: &part.ChecksumSHA1,
SHA256: &part.ChecksumSHA256,
CRC64NVME: &part.ChecksumCRC64NVME,
SHA512: &part.ChecksumSHA512,
MD5: &part.ChecksumMD5,
XXHASH64: &part.ChecksumXXHASH64,
XXHASH3: &part.ChecksumXXHASH3,
XXHASH128: &part.ChecksumXXHASH128,
})
}
func completedPartFromPart(part types.Part) types.CompletedPart {
return types.CompletedPart{
ETag: part.ETag,
PartNumber: part.PartNumber,
ChecksumCRC32: part.ChecksumCRC32,
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumMD5: part.ChecksumMD5,
ChecksumXXHASH64: part.ChecksumXXHASH64,
ChecksumXXHASH3: part.ChecksumXXHASH3,
ChecksumXXHASH128: part.ChecksumXXHASH128,
}
}
func getPutObjectChecksum(out *s3.PutObjectOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func setPutObjectChecksum(in *s3.PutObjectInput, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &in.ChecksumCRC32,
CRC32C: &in.ChecksumCRC32C,
SHA1: &in.ChecksumSHA1,
SHA256: &in.ChecksumSHA256,
CRC64NVME: &in.ChecksumCRC64NVME,
SHA512: &in.ChecksumSHA512,
MD5: &in.ChecksumMD5,
XXHASH64: &in.ChecksumXXHASH64,
XXHASH3: &in.ChecksumXXHASH3,
XXHASH128: &in.ChecksumXXHASH128,
}, checksum)
}
func getGetObjectChecksum(out *s3.GetObjectOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func getHeadObjectChecksum(out *s3.HeadObjectOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func getObjectAttributesChecksum(out *types.Checksum, algo types.ChecksumAlgorithm) *string {
if out == nil {
return nil
}
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func getUploadPartChecksum(out *s3.UploadPartOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func setUploadPartChecksum(in *s3.UploadPartInput, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &in.ChecksumCRC32,
CRC32C: &in.ChecksumCRC32C,
SHA1: &in.ChecksumSHA1,
SHA256: &in.ChecksumSHA256,
CRC64NVME: &in.ChecksumCRC64NVME,
SHA512: &in.ChecksumSHA512,
MD5: &in.ChecksumMD5,
XXHASH64: &in.ChecksumXXHASH64,
XXHASH3: &in.ChecksumXXHASH3,
XXHASH128: &in.ChecksumXXHASH128,
}, checksum)
}
func getCompleteMultipartUploadChecksum(out *s3.CompleteMultipartUploadOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func setCompleteMultipartUploadChecksum(in *s3.CompleteMultipartUploadInput, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &in.ChecksumCRC32,
CRC32C: &in.ChecksumCRC32C,
SHA1: &in.ChecksumSHA1,
SHA256: &in.ChecksumSHA256,
CRC64NVME: &in.ChecksumCRC64NVME,
SHA512: &in.ChecksumSHA512,
MD5: &in.ChecksumMD5,
XXHASH64: &in.ChecksumXXHASH64,
XXHASH3: &in.ChecksumXXHASH3,
XXHASH128: &in.ChecksumXXHASH128,
}, checksum)
}
func getCopyObjectChecksum(result *types.CopyObjectResult, algo types.ChecksumAlgorithm) *string {
if result == nil {
return nil
}
return getChecksum(algo, checksumFields{
CRC32: &result.ChecksumCRC32,
CRC32C: &result.ChecksumCRC32C,
SHA1: &result.ChecksumSHA1,
SHA256: &result.ChecksumSHA256,
CRC64NVME: &result.ChecksumCRC64NVME,
SHA512: &result.ChecksumSHA512,
MD5: &result.ChecksumMD5,
XXHASH64: &result.ChecksumXXHASH64,
XXHASH3: &result.ChecksumXXHASH3,
XXHASH128: &result.ChecksumXXHASH128,
})
}
func getUploadPartCopyChecksum(result *types.CopyPartResult, algo types.ChecksumAlgorithm) *string {
if result == nil {
return nil
}
return getChecksum(algo, checksumFields{
CRC32: &result.ChecksumCRC32,
CRC32C: &result.ChecksumCRC32C,
SHA1: &result.ChecksumSHA1,
SHA256: &result.ChecksumSHA256,
CRC64NVME: &result.ChecksumCRC64NVME,
SHA512: &result.ChecksumSHA512,
MD5: &result.ChecksumMD5,
XXHASH64: &result.ChecksumXXHASH64,
XXHASH3: &result.ChecksumXXHASH3,
XXHASH128: &result.ChecksumXXHASH128,
})
}
// mp1 needs to be the response from the server
// mp2 needs to be the expected values
// The keys from the server are always converted to lowercase
@@ -921,7 +1272,8 @@ func compareDelObjects(list1, list2 []types.DeletedObject) bool {
func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) {
partSize := size / partCount
var hash hash.Hash
var objHasher hash.Hash
var partHasher hash.Hash
cfg := new(mpCfg)
for _, opt := range opts {
@@ -930,17 +1282,34 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
hash = crc32.NewIEEE()
objHasher = crc32.NewIEEE()
case types.ChecksumAlgorithmCrc32c:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
objHasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case types.ChecksumAlgorithmMd5:
objHasher = md5.New()
case types.ChecksumAlgorithmSha1:
hash = sha1.New()
objHasher = sha1.New()
case types.ChecksumAlgorithmSha256:
hash = sha256.New()
objHasher = sha256.New()
case types.ChecksumAlgorithmSha512:
objHasher = sha512.New()
case types.ChecksumAlgorithmCrc64nvme:
hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
objHasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
case types.ChecksumAlgorithmXxhash64:
objHasher = xxhash.New()
case types.ChecksumAlgorithmXxhash3:
objHasher = xxh3.New()
case types.ChecksumAlgorithmXxhash128:
objHasher = xxh3.New128()
default:
hash = sha256.New()
objHasher = sha256.New()
}
if cfg.checksumAlgorithm != "" {
partHasher, err = NewHasher(cfg.checksumAlgorithm)
if err != nil {
return nil, "", err
}
}
for partNumber := int64(1); partNumber <= partCount; partNumber++ {
@@ -952,18 +1321,28 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
partBuffer := make([]byte, partEnd-partStart+1)
rand.Read(partBuffer)
hash.Write(partBuffer)
objHasher.Write(partBuffer)
if partHasher != nil {
partHasher.Write(partBuffer)
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
pn := int32(partNumber)
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
input := &s3.UploadPartInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: &pn,
ChecksumAlgorithm: cfg.checksumAlgorithm,
})
}
if partHasher != nil {
partChecksum := base64.StdEncoding.EncodeToString(partHasher.Sum(nil))
setUploadPartChecksum(input, cfg.checksumAlgorithm, &partChecksum)
partHasher.Reset()
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := client.UploadPart(ctx, input)
cancel()
if err != nil {
return parts, "", err
@@ -975,22 +1354,11 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
Size: &partSize,
}
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
part.ChecksumCRC32 = out.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
part.ChecksumCRC32C = out.ChecksumCRC32C
case types.ChecksumAlgorithmSha1:
part.ChecksumSHA1 = out.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
part.ChecksumSHA256 = out.ChecksumSHA256
case types.ChecksumAlgorithmCrc64nvme:
part.ChecksumCRC64NVME = out.ChecksumCRC64NVME
}
setPartChecksum(&part, cfg.checksumAlgorithm, getUploadPartChecksum(out, cfg.checksumAlgorithm))
parts = append(parts, part)
}
sum := hash.Sum(nil)
sum := objHasher.Sum(nil)
if cfg.checksumAlgorithm == "" {
csum = hex.EncodeToString(sum[:])
@@ -1277,6 +1645,26 @@ func createObjVersions(client *s3.Client, bucket, object string, count int, opts
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmSha256,
}
case r.res.ChecksumSHA512 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmSha512,
}
case r.res.ChecksumMD5 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmMd5,
}
case r.res.ChecksumXXHASH64 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmXxhash64,
}
case r.res.ChecksumXXHASH3 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmXxhash3,
}
case r.res.ChecksumXXHASH128 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmXxhash128,
}
}
versions = append(versions, version)
@@ -2012,14 +2400,26 @@ func lockObject(client *s3.Client, mode objectLockMode, bucket, object, versionI
func NewHasher(algo types.ChecksumAlgorithm) (hash.Hash, error) {
var hasher hash.Hash
switch algo {
case types.ChecksumAlgorithmMd5:
hasher = md5.New()
case types.ChecksumAlgorithmSha256:
hasher = sha256.New()
case types.ChecksumAlgorithmSha512:
hasher = sha512.New()
case types.ChecksumAlgorithmSha1:
hasher = sha1.New()
case types.ChecksumAlgorithmCrc32:
hasher = crc32.NewIEEE()
case types.ChecksumAlgorithmCrc32c:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case types.ChecksumAlgorithmCrc64nvme:
hasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
case types.ChecksumAlgorithmXxhash64:
hasher = xxhash.New()
case types.ChecksumAlgorithmXxhash3:
hasher = xxh3.New()
case types.ChecksumAlgorithmXxhash128:
hasher = xxh3.New128()
default:
return nil, fmt.Errorf("unsupported hash algorithm: %s", algo)
}
@@ -2027,6 +2427,28 @@ func NewHasher(algo types.ChecksumAlgorithm) (hash.Hash, error) {
return hasher, nil
}
func wrongChecksumForAlgorithm(algo types.ChecksumAlgorithm) (string, error) {
var size int
switch algo {
case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c:
size = 4
case types.ChecksumAlgorithmCrc64nvme, types.ChecksumAlgorithmXxhash64, types.ChecksumAlgorithmXxhash3:
size = 8
case types.ChecksumAlgorithmMd5, types.ChecksumAlgorithmXxhash128:
size = 16
case types.ChecksumAlgorithmSha1:
size = 20
case types.ChecksumAlgorithmSha256:
size = 32
case types.ChecksumAlgorithmSha512:
size = 64
default:
return "", fmt.Errorf("unsupported hash algorithm: %s", algo)
}
return base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0xff}, size)), nil
}
func processCompositeChecksum(hasher hash.Hash, checksum string) error {
data, err := base64.StdEncoding.DecodeString(checksum)
if err != nil {