mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-20 06:06:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums

Browse Source 
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
niksis02
2026-05-04 08:50:39 -07:00
committed by Ben McClelland
parent fbe2a4ba10
commit d2fa265fb8
39 changed files with 1811 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/CompleteMultipartUpload.go
+107 -109
View File
@@ -518,6 +518,26 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error {
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo),
@@ -534,15 +554,7 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error {
cParts := []types.CompletedPart{}
for _, el := range parts {
cParts = append(cParts, types.CompletedPart{
ETag: el.ETag,
PartNumber: el.PartNumber,
ChecksumCRC32: el.ChecksumCRC32C,
ChecksumCRC32C: el.ChecksumCRC32C,
ChecksumSHA1: el.ChecksumSHA1,
ChecksumSHA256: el.ChecksumSHA256,
ChecksumCRC64NVME: el.ChecksumCRC64NVME,
})
cParts = append(cParts, completedPartFromPart(el))
}
mpInput := &s3.CompleteMultipartUploadInput{
@@ -566,6 +578,16 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error {
mpInput.ChecksumSHA256 = getPtr("invalid_sha256")
case types.ChecksumAlgorithmCrc64nvme:
mpInput.ChecksumCRC64NVME = getPtr("invalid_crc64nvme")
case types.ChecksumAlgorithmSha512:
mpInput.ChecksumSHA512 = getPtr("invalid_sha512")
case types.ChecksumAlgorithmMd5:
mpInput.ChecksumMD5 = getPtr("invalid_md5")
case types.ChecksumAlgorithmXxhash64:
mpInput.ChecksumXXHASH64 = getPtr("invalid_xxhash64")
case types.ChecksumAlgorithmXxhash3:
mpInput.ChecksumXXHASH3 = getPtr("invalid_xxhash3")
case types.ChecksumAlgorithmXxhash128:
mpInput.ChecksumXXHASH128 = getPtr("invalid_xxhash128")
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
@@ -608,6 +630,26 @@ func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error {
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo),
withChecksumType(el.t))
@@ -623,34 +665,23 @@ func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error {
cParts := []types.CompletedPart{}
for _, el := range parts {
cParts = append(cParts, types.CompletedPart{
ETag: el.ETag,
PartNumber: el.PartNumber,
ChecksumCRC32: el.ChecksumCRC32,
ChecksumCRC32C: el.ChecksumCRC32C,
ChecksumSHA1: el.ChecksumSHA1,
ChecksumSHA256: el.ChecksumSHA256,
ChecksumCRC64NVME: el.ChecksumCRC64NVME,
})
cParts = append(cParts, completedPartFromPart(el))
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
input := &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: cParts,
},
// Provide one of the parts checksum. In any case
// the final checksum will differ from one of the parts checksum
ChecksumCRC32: cParts[0].ChecksumCRC32,
ChecksumCRC32C: cParts[0].ChecksumCRC32C,
ChecksumSHA1: cParts[0].ChecksumSHA1,
ChecksumSHA256: cParts[0].ChecksumSHA256,
ChecksumCRC64NVME: cParts[0].ChecksumCRC64NVME,
ChecksumType: el.t,
})
ChecksumType: el.t,
}
// Provide one of the part checksums. The final checksum will differ
// from a single part checksum.
setCompleteMultipartUploadChecksum(input, el.algo, getCompletedPartChecksum(cParts[0], el.algo))
_, err = s3client.CompleteMultipartUpload(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
return err
@@ -848,6 +879,11 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
types.ChecksumAlgorithmCrc32c,
types.ChecksumAlgorithmSha1,
types.ChecksumAlgorithmSha256,
types.ChecksumAlgorithmSha512,
types.ChecksumAlgorithmMd5,
types.ChecksumAlgorithmXxhash64,
types.ChecksumAlgorithmXxhash3,
types.ChecksumAlgorithmXxhash128,
} {
mp, err := createMp(s3client, bucket, obj, withChecksumType(types.ChecksumTypeComposite), withChecksum(algo))
if err != nil {
@@ -867,29 +903,13 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
completeParts := make([]types.CompletedPart, 0, len(parts))
for _, part := range parts {
switch algo {
case types.ChecksumAlgorithmCrc32:
err = processCompositeChecksum(hasher, getString(part.ChecksumCRC32))
case types.ChecksumAlgorithmCrc32c:
err = processCompositeChecksum(hasher, getString(part.ChecksumCRC32C))
case types.ChecksumAlgorithmSha1:
err = processCompositeChecksum(hasher, getString(part.ChecksumSHA1))
case types.ChecksumAlgorithmSha256:
err = processCompositeChecksum(hasher, getString(part.ChecksumSHA256))
}
err = processCompositeChecksum(hasher, getString(getPartChecksum(part, algo)))
if err != nil {
return fmt.Errorf("test %v failed: %s", i, err)
}
completeParts = append(completeParts, types.CompletedPart{
ETag: part.ETag,
PartNumber: part.PartNumber,
ChecksumCRC32: part.ChecksumCRC32,
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
})
completeParts = append(completeParts, completedPartFromPart(part))
}
checksum := fmt.Sprintf("%s-%v", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), len(parts))
@@ -903,16 +923,7 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
UploadId: mp.UploadId,
}
switch algo {
case types.ChecksumAlgorithmCrc32:
completeMpInput.ChecksumCRC32 = &checksum
case types.ChecksumAlgorithmCrc32c:
completeMpInput.ChecksumCRC32C = &checksum
case types.ChecksumAlgorithmSha1:
completeMpInput.ChecksumSHA1 = &checksum
case types.ChecksumAlgorithmSha256:
completeMpInput.ChecksumSHA256 = &checksum
}
setCompleteMultipartUploadChecksum(completeMpInput, algo, &checksum)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.CompleteMultipartUpload(ctx, completeMpInput)
@@ -921,17 +932,7 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e
return fmt.Errorf("test %v failed: %s", i, err)
}
var gotSum string
switch algo {
case types.ChecksumAlgorithmCrc32:
gotSum = getString(res.ChecksumCRC32)
case types.ChecksumAlgorithmCrc32c:
gotSum = getString(res.ChecksumCRC32C)
case types.ChecksumAlgorithmSha1:
gotSum = getString(res.ChecksumSHA1)
case types.ChecksumAlgorithmSha256:
gotSum = getString(res.ChecksumSHA256)
}
gotSum := getString(getCompleteMultipartUploadChecksum(res, algo))
if gotSum != checksum {
return fmt.Errorf("test %v failed: expected the final checksum to be %s, instead got %s", i, checksum, gotSum)
@@ -947,20 +948,27 @@ func CompleteMultipartUpload_invalid_final_composite_checksum(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for i, test := range []struct {
algo types.ChecksumAlgorithm
crc32 *string
crc32c *string
sha1 *string
sha256 *string
algo types.ChecksumAlgorithm
checksum *string
}{
{types.ChecksumAlgorithmCrc32, getPtr("invalid_checksum"), nil, nil, nil},
{types.ChecksumAlgorithmCrc32, getPtr("ImIEBA==-smth"), nil, nil, nil},
{types.ChecksumAlgorithmCrc32c, nil, getPtr("invalid_checksum"), nil, nil},
{types.ChecksumAlgorithmCrc32c, nil, getPtr("AQIDBA==-12a"), nil, nil},
{types.ChecksumAlgorithmSha1, nil, nil, getPtr("invalid_checksum"), nil},
{types.ChecksumAlgorithmSha1, nil, nil, getPtr("2jmj7l5rSw0yVb/vlWAYkK/YBwk=-10-20"), nil},
{types.ChecksumAlgorithmSha256, nil, nil, nil, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha256, nil, nil, nil, getPtr("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=--3")},
{types.ChecksumAlgorithmCrc32, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmCrc32, getPtr("ImIEBA==-smth")},
{types.ChecksumAlgorithmCrc32c, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmCrc32c, getPtr("AQIDBA==-12a")},
{types.ChecksumAlgorithmSha1, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha1, getPtr("2jmj7l5rSw0yVb/vlWAYkK/YBwk=-10-20")},
{types.ChecksumAlgorithmSha256, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha256, getPtr("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=--3")},
{types.ChecksumAlgorithmSha512, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmSha512, getPtr("z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==-bad")},
{types.ChecksumAlgorithmMd5, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmMd5, getPtr("1B2M2Y8AsgTpgAmY7PhCfg==-bad")},
{types.ChecksumAlgorithmXxhash64, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmXxhash64, getPtr("70bbN1HY6Zk=-bad")},
{types.ChecksumAlgorithmXxhash3, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmXxhash3, getPtr("LQaABTjTlMI=-bad")},
{types.ChecksumAlgorithmXxhash128, getPtr("invalid_checksum")},
{types.ChecksumAlgorithmXxhash128, getPtr("maoG0wFHmNhgAcMkRo1Jfw==-bad")},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(test.algo), withChecksumType(types.ChecksumTypeComposite))
if err != nil {
@@ -975,29 +983,20 @@ func CompleteMultipartUpload_invalid_final_composite_checksum(s *S3Conf) error {
completeParts := make([]types.CompletedPart, 0, len(parts))
for _, part := range parts {
completeParts = append(completeParts, types.CompletedPart{
ETag: part.ETag,
PartNumber: part.PartNumber,
ChecksumCRC32: part.ChecksumCRC32,
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
})
completeParts = append(completeParts, completedPartFromPart(part))
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
input := &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: completeParts,
},
ChecksumCRC32: test.crc32,
ChecksumCRC32C: test.crc32c,
ChecksumSHA1: test.sha1,
ChecksumSHA256: test.sha256,
})
}
setCompleteMultipartUploadChecksum(input, test.algo, test.checksum)
_, err = s3client.CompleteMultipartUpload(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(test.algo))))); err != nil {
return fmt.Errorf("test %v failed: %w", i, err)
@@ -1090,21 +1089,20 @@ func CompleteMultipartUpload_should_ignore_the_final_checksum(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, insted got %v",
*res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, insted got %v",
*res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, insted got %v",
*res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, insted got %v",
*res.ChecksumSHA256)
for _, algo := range []types.ChecksumAlgorithm{
types.ChecksumAlgorithmCrc32,
types.ChecksumAlgorithmCrc32c,
types.ChecksumAlgorithmSha1,
types.ChecksumAlgorithmSha256,
types.ChecksumAlgorithmSha512,
types.ChecksumAlgorithmMd5,
types.ChecksumAlgorithmXxhash64,
types.ChecksumAlgorithmXxhash3,
types.ChecksumAlgorithmXxhash128,
} {
if checksum := getCompleteMultipartUploadChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, insted got %v", algo, *checksum)
}
}
// If no checksum is specified on mp creation, it should default
// to crc64nvme
tests/integration/CopyObject.go
+14 -22
View File
@@ -1461,17 +1461,13 @@ func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error {
if out.CopyObjectResult.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum")
}
if out.CopyObjectResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected empty crc32c checksum")
}
if out.CopyObjectResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected empty sha1 checksum")
}
if out.CopyObjectResult.ChecksumSHA256 != nil {
return fmt.Errorf("expected empty sha256 checksum")
}
if out.CopyObjectResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected empty crc64nvme checksum")
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if algo == types.ChecksumAlgorithmCrc32 {
continue
}
if checksum := getCopyObjectChecksum(out.CopyObjectResult, algo); checksum != nil {
return fmt.Errorf("expected empty %s checksum", algo)
}
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
@@ -1488,17 +1484,13 @@ func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error {
if res.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum")
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected empty crc32c checksum")
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected empty sha1 checksum")
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected empty sha256 checksum")
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected empty crc64nvme checksum")
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if algo == types.ChecksumAlgorithmCrc32 {
continue
}
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected empty %s checksum", algo)
}
}
return nil
tests/integration/CreateMultipartUpload.go
+10
View File
@@ -390,6 +390,11 @@ func CreateMultipartUpload_type_algo_mismatch(s *S3Conf) error {
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc64nvme},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha256},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha512},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmMd5},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash64},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash3},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash128},
} {
_, err := createMp(s3client, bucket, "my-obj", withChecksum(test.algo), withChecksumType(test.chType))
if err := checkApiErr(err, s3err.GetChecksumSchemaMismatchErr(test.algo, test.chType)); err != nil {
@@ -414,6 +419,11 @@ func CreateMultipartUpload_valid_algo_type(s *S3Conf) error {
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha512},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmMd5},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash64},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash3},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash128},
// full object type
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc64nvme},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc32},
tests/integration/GetObject.go
+45 -131
View File
@@ -233,38 +233,13 @@ func GetObject_invalid_parent(s *S3Conf) error {
func GetObject_checksums(s *S3Conf) error {
testName := "GetObject_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -272,7 +247,7 @@ func GetObject_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -281,28 +256,11 @@ func GetObject_checksums(s *S3Conf) error {
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v",
el.key, types.ChecksumTypeFullObject, res.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v",
key, types.ChecksumTypeFullObject, res.ChecksumType)
}
if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32))
}
if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C))
}
if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1))
}
if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256))
}
if getString(res.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC64NVME), getString(res.ChecksumCRC64NVME))
if got, expected := getString(getGetObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got)
}
}
@@ -313,43 +271,31 @@ func GetObject_checksums(s *S3Conf) error {
func GetObject_dir_object_checksum(s *S3Conf) error {
testName := "GetObject_dir_object_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
for i, obj := range []struct {
key string
expectedSum string
for i, test := range []struct {
checksumAlgo types.ChecksumAlgorithm
expectedSum string
}{
{
key: "obj-1/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3/",
expectedSum: "AAAAAAAAAAA=",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
{
key: "obj-4/",
expectedSum: "2jmj7l5rSw0yVb/vlWAYkK/YBwk=",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-5/",
expectedSum: "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{types.ChecksumAlgorithmCrc32, "AAAAAA=="},
{types.ChecksumAlgorithmCrc32c, "AAAAAA=="},
{types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="},
{types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="},
{types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},
{types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="},
{types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="},
{types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="},
{types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="},
{types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, &s3.PutObjectInput{
key := fmt.Sprintf("obj-%v/", i+1)
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj.key,
ChecksumAlgorithm: obj.checksumAlgo,
})
Key: &key,
ChecksumAlgorithm: test.checksumAlgo,
}
setPutObjectChecksum(input, test.checksumAlgo, &test.expectedSum)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, input)
cancel()
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
@@ -358,7 +304,7 @@ func GetObject_dir_object_checksum(s *S3Conf) error {
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &obj.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -368,25 +314,13 @@ func GetObject_dir_object_checksum(s *S3Conf) error {
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("test %v failed: expected the %v object checksum type to be %v, instaed got %v",
i+1, obj.key, types.ChecksumTypeFullObject, res.ChecksumType)
i+1, key, types.ChecksumTypeFullObject, res.ChecksumType)
}
var gotSum *string
switch obj.checksumAlgo {
case types.ChecksumAlgorithmCrc32:
gotSum = res.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
gotSum = res.ChecksumCRC32C
case types.ChecksumAlgorithmCrc64nvme:
gotSum = res.ChecksumCRC64NVME
case types.ChecksumAlgorithmSha1:
gotSum = res.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
gotSum = res.ChecksumSHA256
}
gotSum := getGetObjectChecksum(res, test.checksumAlgo)
if getString(gotSum) != obj.expectedSum {
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, obj.checksumAlgo, obj.expectedSum, getString(gotSum))
if getString(gotSum) != test.expectedSum {
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, test.checksumAlgo, test.expectedSum, getString(gotSum))
}
}
@@ -885,20 +819,10 @@ func GetObject_not_enabled_checksum_mode(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getGetObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -1704,20 +1628,10 @@ func GetObject_ranged_with_checksum_mode(s *S3Conf) error {
testName := "GetObject_ranged_with_checksum_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
checkNoChecksums := func(res *s3.GetObjectOutput) error {
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getGetObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
}
tests/integration/GetObjectAttributes.go
+11 -53
View File
@@ -235,38 +235,13 @@ func GetObjectAttributes_existing_object(s *S3Conf) error {
func GetObjectAttributes_checksums(s *S3Conf) error {
testName := "GetObjectAttributes_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -274,7 +249,7 @@ func GetObjectAttributes_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesChecksum,
},
@@ -288,28 +263,11 @@ func GetObjectAttributes_checksums(s *S3Conf) error {
return fmt.Errorf("expected non-nil checksum in the response")
}
if res.Checksum.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v",
el.key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v",
key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType)
}
if getString(res.Checksum.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.Checksum.ChecksumCRC32))
}
if getString(res.Checksum.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32C), getString(res.Checksum.ChecksumCRC32C))
}
if getString(res.Checksum.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA1), getString(res.Checksum.ChecksumSHA1))
}
if getString(res.Checksum.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA256), getString(res.Checksum.ChecksumSHA256))
}
if getString(res.Checksum.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC64NVME), getString(res.Checksum.ChecksumCRC64NVME))
if got, expected := getString(getObjectAttributesChecksum(res.Checksum, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %s, instead got %s", checksumAlgo, expected, got)
}
}
return nil
tests/integration/HeadObject.go
+18 -72
View File
@@ -150,17 +150,10 @@ func HeadObject_not_enabled_checksum_mode(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -170,38 +163,13 @@ func HeadObject_not_enabled_checksum_mode(s *S3Conf) error {
func HeadObject_checksums(s *S3Conf) error {
testName := "HeadObject_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*200), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*200), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -209,7 +177,7 @@ func HeadObject_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -218,22 +186,10 @@ func HeadObject_checksums(s *S3Conf) error {
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v", key, types.ChecksumTypeFullObject, res.ChecksumType)
}
if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32))
}
if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C))
}
if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1))
}
if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256))
}
if getString(res.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v", getString(out.res.ChecksumCRC64NVME), getString(res.ChecksumCRC64NVME))
if got, expected := getString(getHeadObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got)
}
}
@@ -1311,20 +1267,10 @@ func HeadObject_ranged_with_checksum_mode(s *S3Conf) error {
testName := "HeadObject_ranged_with_checksum_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
checkNoChecksums := func(res *s3.HeadObjectOutput) error {
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
}
tests/integration/ListMultipartUploads.go
+26
View File
@@ -612,6 +612,32 @@ func ListMultipartUploads_with_checksums(s *S3Conf) error {
algo: types.ChecksumAlgorithmCrc64nvme,
t: types.ChecksumTypeFullObject,
},
{
obj: "obj-6",
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
{
obj: "obj-7",
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
{
obj: "obj-8",
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
{
obj: "obj-9",
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
// obj-91 to preserve the lexicographical order
{
obj: "obj-91",
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
} {
key := el.obj
mp, err := createMp(s3client, bucket, key, withChecksum(el.algo), withChecksumType(el.t))
tests/integration/ListObjects.go
+1 -1
View File
@@ -349,7 +349,7 @@ func ListObjects_with_checksum(s *S3Conf) error {
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: el,
}, s3client)
}, s3client, withPutObjectChecksumAlgo(el))
if err != nil {
return err
}
tests/integration/ListObjectsV2.go
+1 -1
View File
@@ -578,7 +578,7 @@ func ListObjectsV2_with_checksum(s *S3Conf) error {
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: el,
}, s3client)
}, s3client, withPutObjectChecksumAlgo(el))
if err != nil {
return err
}
tests/integration/PostObject.go
+5
View File
@@ -1070,6 +1070,11 @@ func PostObject_checksums_success(s *S3Conf) error {
{"x-amz-checksum-crc64nvme", "rsrzr5yYqFU="},
{"x-amz-checksum-sha1", "9I3YU4IIYIFsddVND1hNyGMyenw="},
{"x-amz-checksum-sha256", "kW8AJ6V1B0znKjMXd8NHjWUT94alkb2JLaGld78jNfk="},
{"x-amz-checksum-sha512", "Dh4h7PEF7IU9JNcohnrXBhPCFmOkaTB0sqNhnBvTnWa1iMM3I7tGbHJCToDjymPCSQeKs0e6uUKFAOfuQwWdDQ=="},
{"x-amz-checksum-md5", "63M6AMDJ0zbmVpGjerVCkw=="},
{"x-amz-checksum-xxhash64", "+lb36/ER8bo="},
{"x-amz-checksum-xxhash3", "jw+pSh/pbMQ="},
{"x-amz-checksum-xxhash128", "8BLDqqIWji+ITOsp/JjN/Q=="},
} {
resp, err := sendPostObject(PostRequestConfig{
bucket: bucket,
tests/integration/PutObject.go
+58 -185
View File
@@ -702,98 +702,31 @@ func PutObject_invalid_checksum_header(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for i, el := range []struct {
algo string
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
// CRC32 tests
{
algo: "crc32",
crc32: getPtr(""),
},
{
algo: "crc32",
crc32: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32",
crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32
},
// CRC32C tests
{
algo: "crc32c",
crc32c: getPtr(""),
},
{
algo: "crc32c",
crc32c: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32c",
crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c
},
// SHA1 tests
{
algo: "sha1",
sha1: getPtr(""),
},
{
algo: "sha1",
sha1: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha1",
sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1
},
// SHA256 tests
{
algo: "sha256",
sha256: getPtr(""),
},
{
algo: "sha256",
sha256: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha256",
sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56
},
// CRC64Nvme tests
{
algo: "crc64nvme",
sha256: getPtr(""),
},
{
algo: "crc64nvme",
sha256: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc64nvme",
sha256: getPtr("ZHNhZmRzYWZzZGFmZHNhZg=="), // valid base64 but not crc64nvme
},
} {
_, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
}, s3client)
i := 0
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
// tests against:
// - empty string
// - invalid base64
// - valid base64, but invalid checksum
for _, checksum := range []string{"", "invalid_base64!", "c2RhZnNhZGZzZGFm"} {
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}
setPutObjectChecksum(input, algo, getPtr(checksum))
// FIXME: The error message for PutObject is not properly serialized by the sdk
// References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921
_, err := putObjectWithData(int64((i+1)*100), input, s3client)
i++
// if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil {
// return err
// }
if err := checkSdkApiErr(err, "InvalidRequest"); err != nil {
return err
// FIXME: The error message for PutObject is not properly serialized by the sdk
// References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921
// if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil {
// return err
// }
if err := checkSdkApiErr(err, "InvalidRequest"); err != nil {
return err
}
}
}
@@ -807,62 +740,33 @@ func PutObject_incorrect_checksums(s *S3Conf) error {
obj := "my-obj"
dirObj := "dir-object/"
for i, el := range []struct {
algo types.ChecksumAlgorithm
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
{
algo: types.ChecksumAlgorithmCrc32,
crc32: getPtr("DUoRhQ=="),
},
{
algo: types.ChecksumAlgorithmCrc32c,
crc32c: getPtr("yZRlqg=="),
},
{
algo: types.ChecksumAlgorithmSha1,
sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="),
},
{
algo: types.ChecksumAlgorithmSha256,
sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="),
},
{
algo: types.ChecksumAlgorithmCrc64nvme,
crc64nvme: getPtr("sV264W+gYBI="),
},
} {
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
wrongChecksum, err := wrongChecksumForAlgorithm(algo)
if err != nil {
return err
}
// test for file object
_, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
}, s3client)
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}
setPutObjectChecksum(input, algo, &wrongChecksum)
_, err = putObjectWithData(int64((i+1)*100), input, s3client)
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil {
return err
}
// test for directory object
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &bucket,
Key: &dirObj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
})
input = &s3.PutObjectInput{
Bucket: &bucket,
Key: &dirObj,
}
setPutObjectChecksum(input, algo, &wrongChecksum)
_, err = s3client.PutObject(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil {
return err
}
}
@@ -952,13 +856,20 @@ func PutObject_dir_object_checksums_success(s *S3Conf) error {
{types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="},
{types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="},
{types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},
{types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="},
{types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="},
{types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="},
{types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="},
{types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.PutObject(ctx, &s3.PutObjectInput{
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: getPtr(fmt.Sprintf("obj-%v/", i)),
ChecksumAlgorithm: test.algo,
})
}
setPutObjectChecksum(input, test.algo, &test.checksumValue)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.PutObject(ctx, input)
cancel()
if err != nil {
return err
@@ -968,27 +879,8 @@ func PutObject_dir_object_checksums_success(s *S3Conf) error {
return fmt.Errorf("expected the checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType)
}
switch test.algo {
case types.ChecksumAlgorithmCrc32:
if getString(res.ChecksumCRC32) != test.checksumValue {
return fmt.Errorf("expected the crc32 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32))
}
case types.ChecksumAlgorithmCrc32c:
if getString(res.ChecksumCRC32C) != test.checksumValue {
return fmt.Errorf("expected the crc32c checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32C))
}
case types.ChecksumAlgorithmSha1:
if getString(res.ChecksumSHA1) != test.checksumValue {
return fmt.Errorf("expected the sha1 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA1))
}
case types.ChecksumAlgorithmSha256:
if getString(res.ChecksumSHA256) != test.checksumValue {
return fmt.Errorf("expected the sha256 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA256))
}
case types.ChecksumAlgorithmCrc64nvme:
if getString(res.ChecksumCRC64NVME) != test.checksumValue {
return fmt.Errorf("expected the crc64nvme checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC64NVME))
}
if got := getString(getPutObjectChecksum(res, test.algo)); got != test.checksumValue {
return fmt.Errorf("expected the %s checksum value to be %s, instead got %s", test.algo, test.checksumValue, got)
}
}
@@ -1006,7 +898,7 @@ func PutObject_checksums_success(s *S3Conf) error {
Bucket: &bucket,
Key: &obj,
ChecksumAlgorithm: algo,
}, s3client)
}, s3client, withPutObjectChecksumAlgo(algo))
if err != nil {
return err
}
@@ -1015,27 +907,8 @@ func PutObject_checksums_success(s *S3Conf) error {
return fmt.Errorf("expected the object checksum type to be %v, instead got %v", types.ChecksumTypeFullObject, res.res.ChecksumType)
}
switch algo {
case types.ChecksumAlgorithmCrc32:
if res.res.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum in the response")
}
case types.ChecksumAlgorithmCrc32c:
if res.res.ChecksumCRC32C == nil {
return fmt.Errorf("expected non empty crc32c checksum in the response")
}
case types.ChecksumAlgorithmSha1:
if res.res.ChecksumSHA1 == nil {
return fmt.Errorf("expected non empty sha1 checksum in the response")
}
case types.ChecksumAlgorithmSha256:
if res.res.ChecksumSHA256 == nil {
return fmt.Errorf("expected non empty sha256 checksum in the response")
}
case types.ChecksumAlgorithmCrc64nvme:
if res.res.ChecksumCRC64NVME == nil {
return fmt.Errorf("expected non empty crc64nvme checksum in the response")
}
if getPutObjectChecksum(res.res, algo) == nil {
return fmt.Errorf("expected non empty %s checksum in the response", algo)
}
}
tests/integration/UploadPart.go
+51 -213
View File
@@ -155,95 +155,26 @@ func UploadPart_invalid_checksum_header(s *S3Conf) error {
partNumber := int32(1)
for _, el := range []struct {
algo string
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
// CRC32 tests
{
algo: "crc32",
crc32: getPtr(""),
},
{
algo: "crc32",
crc32: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32",
crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32
},
// CRC32C tests
{
algo: "crc32c",
crc32c: getPtr(""),
},
{
algo: "crc32c",
crc32c: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc32c",
crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c
},
// SHA1 tests
{
algo: "sha1",
sha1: getPtr(""),
},
{
algo: "sha1",
sha1: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha1",
sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1
},
// SHA256 tests
{
algo: "sha256",
sha256: getPtr(""),
},
{
algo: "sha256",
sha256: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "sha256",
sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56
},
// CRC64NVME tests
{
algo: "crc64nvme",
crc64nvme: getPtr(""),
},
{
algo: "crc64nvme",
crc64nvme: getPtr("invalid_base64!"), // invalid base64
},
{
algo: "crc64nvme",
crc64nvme: getPtr("ZHNhZmRzYWZzZGFmZHNhZg=="), // valid base64 but not crc64nvme
},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
PartNumber: &partNumber,
UploadId: mp.UploadId,
})
cancel()
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", el.algo))); err != nil {
return err
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
// tests against:
// - empty string
// - invalid base64
// - valid base64, but invalid checksum
for _, checksum := range []string{"", "invalid_base64!", "c2RhZnNhZGZzZGFm"} {
input := &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
PartNumber: &partNumber,
UploadId: mp.UploadId,
}
setUploadPartChecksum(input, algo, getPtr(checksum))
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.UploadPart(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(checksumHeaderName(algo))); err != nil {
return err
}
}
}
@@ -336,36 +267,13 @@ func UploadPart_incorrect_checksums(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for _, el := range []struct {
algo types.ChecksumAlgorithm
crc32 *string
crc32c *string
sha1 *string
sha256 *string
crc64nvme *string
}{
{
algo: types.ChecksumAlgorithmCrc32,
crc32: getPtr("DUoRhQ=="),
},
{
algo: types.ChecksumAlgorithmCrc32c,
crc32c: getPtr("yZRlqg=="),
},
{
algo: types.ChecksumAlgorithmSha1,
sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="),
},
{
algo: types.ChecksumAlgorithmSha256,
sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="),
},
{
algo: types.ChecksumAlgorithmCrc64nvme,
crc64nvme: getPtr("MN2ofvMjpIQ="),
},
} {
mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo))
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
wrongChecksum, err := wrongChecksumForAlgorithm(algo)
if err != nil {
return err
}
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
@@ -374,20 +282,17 @@ func UploadPart_incorrect_checksums(s *S3Conf) error {
partNumber := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
UploadId: mp.UploadId,
PartNumber: &partNumber,
Body: body,
})
input := &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
Body: body,
}
setUploadPartChecksum(input, algo, &wrongChecksum)
_, err = s3client.UploadPart(ctx, input)
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil {
return err
}
}
@@ -479,6 +384,11 @@ func UploadPart_no_checksum_with_composite_checksum_type(s *S3Conf) error {
types.ChecksumAlgorithmCrc32c,
types.ChecksumAlgorithmSha1,
types.ChecksumAlgorithmSha256,
types.ChecksumAlgorithmSha512,
types.ChecksumAlgorithmMd5,
types.ChecksumAlgorithmXxhash64,
types.ChecksumAlgorithmXxhash3,
types.ChecksumAlgorithmXxhash128,
} {
mp, err := createMp(customClient, bucket, obj, withChecksum(algo), withChecksumType(types.ChecksumTypeComposite))
if err != nil {
@@ -493,8 +403,8 @@ func UploadPart_no_checksum_with_composite_checksum_type(s *S3Conf) error {
})
}
func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Conf) error {
testName := "UploadPart_should_calculate_checksum_if_only_algorithm_is_provided"
func UploadPart_with_checksums_success(s *S3Conf) error {
testName := "UploadPart_with_checksums_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
customClient := s3.NewFromConfig(s.Config(), func(o *s3.Options) {
o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset
@@ -512,6 +422,11 @@ func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Con
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha512},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmMd5},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash64},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash3},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash128},
} {
mp, err := createMp(customClient, bucket, obj, withChecksum(test.chAlgo), withChecksumType(test.chType))
if err != nil {
@@ -527,91 +442,14 @@ func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Con
return fmt.Errorf("expected 1 uploaded part, instaed got %d", len(parts))
}
part := parts[0]
switch test.chAlgo {
case types.ChecksumAlgorithmCrc32:
if getString(part.ChecksumCRC32) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32))
}
case types.ChecksumAlgorithmCrc32c:
if getString(part.ChecksumCRC32C) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32C))
}
case types.ChecksumAlgorithmCrc64nvme:
if getString(part.ChecksumCRC64NVME) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC64NVME))
}
case types.ChecksumAlgorithmSha1:
if getString(part.ChecksumSHA1) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA1))
}
case types.ChecksumAlgorithmSha256:
if getString(part.ChecksumSHA256) != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA256))
}
if got := getString(getPartChecksum(parts[0], test.chAlgo)); got != csum {
return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, got)
}
}
return nil
})
}
func UploadPart_with_checksums_success(s *S3Conf) error {
testName := "UploadPart_with_checksums_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
partNumber := int32(1)
data := make([]byte, i*100)
rand.Read(data)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &obj,
ChecksumAlgorithm: algo,
UploadId: mp.UploadId,
PartNumber: &partNumber,
Body: bytes.NewReader(data),
})
cancel()
if err != nil {
return err
}
switch algo {
case types.ChecksumAlgorithmCrc32:
if res.ChecksumCRC32 == nil {
return fmt.Errorf("expected non empty crc32 checksum in the response")
}
case types.ChecksumAlgorithmCrc32c:
if res.ChecksumCRC32C == nil {
return fmt.Errorf("expected non empty crc32c checksum in the response")
}
case types.ChecksumAlgorithmSha1:
if res.ChecksumSHA1 == nil {
return fmt.Errorf("expected non empty sha1 checksum in the response")
}
case types.ChecksumAlgorithmSha256:
if res.ChecksumSHA256 == nil {
return fmt.Errorf("expected non empty sha256 checksum in the response")
}
case types.ChecksumAlgorithmCrc64nvme:
if res.ChecksumCRC64NVME == nil {
return fmt.Errorf("expected non empty crc64nvme checksum in the response")
}
}
}
return nil
})
}
func UploadPart_non_existing_key(s *S3Conf) error {
testName := "UploadPart_non_existing_key"
partNumber := int32(1)
tests/integration/UploadPartCopy.go
+83 -110
View File
@@ -845,57 +845,50 @@ func UploadPartCopy_conditional_reads(s *S3Conf) error {
func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error {
testName := "UploadPartCopy_should_copy_the_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
srcObj := "source-object"
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
obj := fmt.Sprintf("my-obj-%v", i+1)
srcObj := fmt.Sprintf("source-object-%v", i+1)
mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32))
if err != nil {
return err
}
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
out, err := putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: types.ChecksumAlgorithmCrc32,
}, s3client)
if err != nil {
return err
}
out, err := putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: algo,
}, s3client, withPutObjectChecksumAlgo(algo))
if err != nil {
return err
}
partNumber := int32(1)
partNumber := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
if getString(res.CopyPartResult.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.CopyPartResult.ChecksumCRC32))
}
if res.CopyPartResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32C)
}
if res.CopyPartResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA1)
}
if res.CopyPartResult.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA256)
}
if res.CopyPartResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC64NVME)
if got, expected := getString(getUploadPartCopyChecksum(res.CopyPartResult, algo)), getString(getPutObjectChecksum(out.res, algo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", algo, expected, got)
}
for _, otherAlgo := range types.ChecksumAlgorithmCrc32.Values() {
if otherAlgo == algo {
continue
}
if checksum := getUploadPartCopyChecksum(res.CopyPartResult, otherAlgo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", otherAlgo, *checksum)
}
}
}
return nil
@@ -937,25 +930,10 @@ func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error {
return err
}
if res.CopyPartResult.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32)
}
if res.CopyPartResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32C)
}
if res.CopyPartResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA1)
}
if res.CopyPartResult.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA256)
}
if res.CopyPartResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getUploadPartCopyChecksum(res.CopyPartResult, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -965,56 +943,51 @@ func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error {
func UploadPartCopy_should_calculate_the_checksum(s *S3Conf) error {
testName := "UploadPartCopy_should_calculate_the_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
srcObj := "source-object"
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
obj := fmt.Sprintf("my-obj-%v", i+1)
srcObj := fmt.Sprintf("source-object-%v", i+1)
mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha256))
if err != nil {
return err
}
mp, err := createMp(s3client, bucket, obj, withChecksum(algo))
if err != nil {
return err
}
_, err = putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: types.ChecksumAlgorithmSha1, // different from the mp checksum (sha256)
}, s3client)
if err != nil {
return err
}
_, err = putObjectWithData(300, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ChecksumAlgorithm: types.ChecksumAlgorithmSha1,
}, s3client)
if err != nil {
return err
}
partNumber := int32(1)
partNumber := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
PartNumber: &partNumber,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
if res.CopyPartResult.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32)
}
if res.CopyPartResult.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC32C)
}
if res.CopyPartResult.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v",
*res.CopyPartResult.ChecksumCRC64NVME)
}
if res.CopyPartResult.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v",
*res.CopyPartResult.ChecksumSHA1)
}
if getString(res.CopyPartResult.ChecksumSHA256) == "" {
return fmt.Errorf("expected non empty sha256 checksum")
for _, otherAlgo := range types.ChecksumAlgorithmCrc32.Values() {
checksum := getUploadPartCopyChecksum(res.CopyPartResult, otherAlgo)
if otherAlgo == algo {
if getString(checksum) == "" {
return fmt.Errorf("expected non empty %s checksum", algo)
}
continue
}
if checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", otherAlgo, *checksum)
}
}
}
return nil
tests/integration/group-tests.go
-2
View File
@@ -445,7 +445,6 @@ func TestUploadPart(ts *TestState) {
ts.Run(UploadPart_incorrect_checksums)
ts.Run(UploadPart_no_checksum_with_full_object_checksum_type)
ts.Run(UploadPart_no_checksum_with_composite_checksum_type)
ts.Run(UploadPart_should_calculate_checksum_if_only_algorithm_is_provided)
ts.Run(UploadPart_with_checksums_success)
}
ts.Run(UploadPart_success)
@@ -1589,7 +1588,6 @@ func GetIntTests() IntTests {
"UploadPart_incorrect_checksums": UploadPart_incorrect_checksums,
"UploadPart_no_checksum_with_full_object_checksum_type": UploadPart_no_checksum_with_full_object_checksum_type,
"UploadPart_no_checksum_with_composite_checksum_type": UploadPart_no_checksum_with_composite_checksum_type,
"UploadPart_should_calculate_checksum_if_only_algorithm_is_provided": UploadPart_should_calculate_checksum_if_only_algorithm_is_provided,
"UploadPart_with_checksums_success": UploadPart_with_checksums_success,
"UploadPart_success": UploadPart_success,
"UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket,
tests/integration/signed_streaming_payload_trailer.go
+27 -23
View File
@@ -108,24 +108,17 @@ func SignedStreamingPayloadTrailer_invalid_checksum(s *S3Conf) error {
testName := "SignedStreamingPayloadTrailer_invalid_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
trailerHdr string
trailingChecksum string
}{
{"x-amz-checksum-crc32", "x-amz-checksum-crc32:invalid"},
{"x-amz-checksum-crc32c", "x-amz-checksum-crc32c:invalid"},
{"x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:invalid"},
{"x-amz-checksum-sha1", "x-amz-checksum-sha1:invalid"},
{"x-amz-checksum-sha256", "x-amz-checksum-sha256:invalid"},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": test.trailerHdr,
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
trailerHdr := checksumHeaderName(algo)
trailingChecksum := fmt.Sprintf("%s:invalid", trailerHdr)
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": trailerHdr,
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(test.trailerHdr), apiErr); err != nil {
if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(trailerHdr), apiErr); err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
}
@@ -139,18 +132,24 @@ func SignedStreamingPayloadTrailer_bad_digest(s *S3Conf) error {
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
object := "my-object"
for i, test := range []struct {
algo types.ChecksumAlgorithm
trailerHdr string
trailingChecksum string
algo types.ChecksumAlgorithm
checksum string
}{
{types.ChecksumAlgorithmCrc32, "x-amz-checksum-crc32", "x-amz-checksum-crc32:NhCmhg=="},
{types.ChecksumAlgorithmCrc32c, "x-amz-checksum-crc32c", "x-amz-checksum-crc32c:+Cy97w=="},
{types.ChecksumAlgorithmCrc64nvme, "x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:QFRKMGE3tuw="},
{types.ChecksumAlgorithmSha1, "x-amz-checksum-sha1", "x-amz-checksum-sha1:qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{types.ChecksumAlgorithmSha256, "x-amz-checksum-sha256", "x-amz-checksum-sha256:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
{types.ChecksumAlgorithmCrc32, "NhCmhg=="},
{types.ChecksumAlgorithmCrc32c, "+Cy97w=="},
{types.ChecksumAlgorithmCrc64nvme, "QFRKMGE3tuw="},
{types.ChecksumAlgorithmSha1, "qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{types.ChecksumAlgorithmSha256, "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
{types.ChecksumAlgorithmSha512, "mD1D3f9tqQ9qXTthckRqH/4ii4A/5k/dXc+rVkYHioloUf6C9iPJ1uVlSz0vNjoE7BfPtitgdDepx8Ey1RHlIg=="},
{types.ChecksumAlgorithmMd5, "Q3uTDbhLgHnC3YBKcZNrXw=="},
{types.ChecksumAlgorithmXxhash64, "dVz0EuroMyA="},
{types.ChecksumAlgorithmXxhash3, "OHmyyhTeg1E="},
{types.ChecksumAlgorithmXxhash128, "6fhA2KZsN830zx0e5P183w=="},
} {
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": test.trailerHdr,
trailerHdr := checksumHeaderName(test.algo)
trailingChecksum := fmt.Sprintf("%s:%s", trailerHdr, test.checksum)
_, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(trailingChecksum), withCustomHeaders(map[string]string{
"x-amz-trailer": trailerHdr,
}))
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
@@ -178,6 +177,11 @@ func SignedStreamingPayloadTrailer_success(s *S3Conf) error {
{"x-amz-checksum-crc64nvme", "dYnI3/Fh0gM="},
{"x-amz-checksum-sha1", "8O8FwCfmd5fCbCBvH09mrKMVoHU="},
{"x-amz-checksum-sha256", "OoSow5X4zTIPl27MtdFdYT+9O3C367C75+Cb2MFtRBc="},
{"x-amz-checksum-sha512", "uiz+VuENyLjzFxWrdmbN/NdIdldj/V3saJF6FsckcYu6xu26fM0CfAaTySaoFJmIdI2m5wbYMJxtShQ1PXk3Tg=="},
{"x-amz-checksum-md5", "EtWtthXtCk3RxUiXKw+ydw=="},
{"x-amz-checksum-xxhash64", "McmxUfNFLUs="},
{"x-amz-checksum-xxhash3", "tLh+aNltTM4="},
{"x-amz-checksum-xxhash128", "MXqrLVobrQkTBf82+7i9AQ=="},
} {
headers, apiErr, err := testSignedStreamingObjectPut(
s,
tests/integration/unsigned_streaming_payload_trailer.go
+30 -9
View File
@@ -134,6 +134,11 @@ func UnsignedStreamingPayloadTrailer_multiple_checksum_headers(s *S3Conf) error
{"crc64nvme", "QFRKMGE3tuw="},
{"sha1", "qvTGHdzF6KLavt4PO0gs2a6pQ00="},
{"sha256", "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="},
{"sha512", "uiz+VuENyLjzFxWrdmbN/NdIdldj/V3saJF6FsckcYu6xu26fM0CfAaTySaoFJmIdI2m5wbYMJxtShQ1PXk3Tg=="},
{"md5", "EtWtthXtCk3RxUiXKw+ydw=="},
{"xxhash64", "McmxUfNFLUs="},
{"xxhash3", "tLh+aNltTM4="},
{"xxhash128", "MXqrLVobrQkTBf82+7i9AQ=="},
} {
reqHeaders := map[string]string{
"x-amz-decoded-content-length": "5",
@@ -348,6 +353,11 @@ func UnsignedStreamingPayloadTrailer_no_payload_trailer_only_headers(s *S3Conf)
{"crc64nvme", "SmzZ/LTp1CA="},
{"sha1", "L7XhNBn8iSRoZeejJPR27GJOh0A="},
{"sha256", "fRpUEnsiJQL1t5tfsIAwYRUqRPkrN+I8ZSe69mXU2po="},
{"sha512", "1xakGIVptoqxtt+sF45XARTN8Oo6HMDjFIbD5BJBvGp2Qk6MN6sm8Jb8he+YhsjLY0GH9P3f9kX7CZ8f9UxrjA=="},
{"md5", "esZsDxSN6VGbi9JkMSxNZA=="},
{"xxhash64", "GGCUDikCgi0="},
{"xxhash3", "WkDcP9RMBS8="},
{"xxhash128", "Kq/YOGmlnDE/55jA7aptxg=="},
} {
csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key)
reqHeaders := map[string]string{
@@ -387,6 +397,11 @@ func UnsignedStreamingPayloadTrailer_success_both_sdk_algo_and_trailer(s *S3Conf
{"crc64nvme", "SmzZ/LTp1CA="},
{"sha1", "L7XhNBn8iSRoZeejJPR27GJOh0A="},
{"sha256", "fRpUEnsiJQL1t5tfsIAwYRUqRPkrN+I8ZSe69mXU2po="},
{"sha512", "1xakGIVptoqxtt+sF45XARTN8Oo6HMDjFIbD5BJBvGp2Qk6MN6sm8Jb8he+YhsjLY0GH9P3f9kX7CZ8f9UxrjA=="},
{"md5", "esZsDxSN6VGbi9JkMSxNZA=="},
{"xxhash64", "GGCUDikCgi0="},
{"xxhash3", "WkDcP9RMBS8="},
{"xxhash128", "Kq/YOGmlnDE/55jA7aptxg=="},
} {
csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key)
reqHeaders := map[string]string{
@@ -508,23 +523,29 @@ func UnsignedStreamingPayloadTrailer_UploadPart_success_with_trailer(s *S3Conf)
object := "my-object"
for i, test := range []struct {
key string
algo types.ChecksumAlgorithm
value string
}{
{"crc32", "QWaN2w=="},
{"crc32c", "R/I7iQ=="},
{"crc64nvme", "dPVWc2vU1+Q="},
{"sha1", "YR/1TvTYOJz5gtqVFoBJBtmTibY="},
{"sha256", "eXuwq/95jXIAr3aF3KeQHt/8Ur8mUA1b2XKCZY7iQVI="},
{types.ChecksumAlgorithmCrc32, "QWaN2w=="},
{types.ChecksumAlgorithmCrc32c, "R/I7iQ=="},
{types.ChecksumAlgorithmCrc64nvme, "dPVWc2vU1+Q="},
{types.ChecksumAlgorithmSha1, "YR/1TvTYOJz5gtqVFoBJBtmTibY="},
{types.ChecksumAlgorithmSha256, "eXuwq/95jXIAr3aF3KeQHt/8Ur8mUA1b2XKCZY7iQVI="},
{types.ChecksumAlgorithmSha512, "6tQCd6X50h2wXt5deAY0eKF5Xb1LLbSvkZt+Bqczz8bd1+rH+VYZSgWhjOG4zJ41K0kLmDQxuDyOeHMKqKQGNA=="},
{types.ChecksumAlgorithmMd5, "Mb+5cwrlGvc5U7pyDIZg1w=="},
{types.ChecksumAlgorithmXxhash64, "vYOmFpsGXtY="},
{types.ChecksumAlgorithmXxhash3, "9J6xx21X+f8="},
{types.ChecksumAlgorithmXxhash128, "menoCZOCZ2Acv4XMxaUR9w=="},
} {
mp, err := createMp(s3client, bucket, object, withChecksum(types.ChecksumAlgorithm(strings.ToUpper(test.key))))
mp, err := createMp(s3client, bucket, object, withChecksum(test.algo))
if err != nil {
return err
}
csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key)
csumHdr := checksumHeaderName(test.algo)
reqHeaders := map[string]string{
"x-amz-decoded-content-length": "10",
"x-amz-sdk-checksum-algorithm": test.key,
"x-amz-sdk-checksum-algorithm": strings.ToLower(string(test.algo)),
"x-amz-trailer": csumHdr,
}
body := bytes.NewBuffer([]byte("A\r\ndummy data\r\n0\r\n"))
tests/integration/utils.go
+449 -27
View File
@@ -22,6 +22,7 @@ import (
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
@@ -53,7 +54,9 @@ import (
"github.com/aws/smithy-go"
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
)
@@ -547,21 +550,46 @@ func hasPrefixName(prefixes []types.CommonPrefix, names []string) bool {
return true
}
type putObjectCfg struct {
checksumAlgorithm types.ChecksumAlgorithm
}
type putObjectOpt func(*putObjectCfg)
func withPutObjectChecksumAlgo(algo types.ChecksumAlgorithm) putObjectOpt {
return func(poc *putObjectCfg) { poc.checksumAlgorithm = algo }
}
type putObjectOutput struct {
csum [32]byte
data []byte
res *s3.PutObjectOutput
}
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) {
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client, opts ...putObjectOpt) (*putObjectOutput, error) {
cfg := &putObjectCfg{}
for _, opt := range opts {
opt(cfg)
}
var csum [32]byte
var data []byte
if input.Body == nil && lgth != 0 {
data = make([]byte, lgth)
rand.Read(data)
csum = sha256.Sum256(data)
r := bytes.NewReader(data)
input.Body = r
if cfg.checksumAlgorithm != "" {
hasher, err := NewHasher(cfg.checksumAlgorithm)
if err != nil {
return nil, err
}
hasher.Write(data)
sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
setPutObjectChecksum(input, cfg.checksumAlgorithm, &sum)
}
input.Body = bytes.NewReader(data)
}
ctx, cancel := context.WithTimeout(context.Background(), longTimeout)
@@ -713,6 +741,36 @@ func compareParts(parts1, parts2 []types.Part) bool {
return false
}
}
if prt.ChecksumSHA512 != nil {
if *prt.ChecksumSHA512 != getString(parts2[i].ChecksumSHA512) {
fmt.Printf("sha512 checksums are not equal, %v != %v\n", *prt.ChecksumSHA512, getString(parts2[i].ChecksumSHA512))
return false
}
}
if prt.ChecksumMD5 != nil {
if *prt.ChecksumMD5 != getString(parts2[i].ChecksumMD5) {
fmt.Printf("md5 checksums are not equal, %v != %v\n", *prt.ChecksumMD5, getString(parts2[i].ChecksumMD5))
return false
}
}
if prt.ChecksumXXHASH64 != nil {
if *prt.ChecksumXXHASH64 != getString(parts2[i].ChecksumXXHASH64) {
fmt.Printf("xxhash64 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH64, getString(parts2[i].ChecksumXXHASH64))
return false
}
}
if prt.ChecksumXXHASH3 != nil {
if *prt.ChecksumXXHASH3 != getString(parts2[i].ChecksumXXHASH3) {
fmt.Printf("xxhash3 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH3, getString(parts2[i].ChecksumXXHASH3))
return false
}
}
if prt.ChecksumXXHASH128 != nil {
if *prt.ChecksumXXHASH128 != getString(parts2[i].ChecksumXXHASH128) {
fmt.Printf("xxhash128 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH128, getString(parts2[i].ChecksumXXHASH128))
return false
}
}
}
return true
}
@@ -775,6 +833,299 @@ func getPtr[T any](str T) *T {
return &str
}
func checksumHeaderName(algo types.ChecksumAlgorithm) string {
return fmt.Sprintf("x-amz-checksum-%s", strings.ToLower(string(algo)))
}
type checksumFields struct {
CRC32 **string
CRC32C **string
SHA1 **string
SHA256 **string
CRC64NVME **string
SHA512 **string
MD5 **string
XXHASH64 **string
XXHASH3 **string
XXHASH128 **string
}
func selectChecksum(algo types.ChecksumAlgorithm, fields checksumFields) **string {
switch algo {
case types.ChecksumAlgorithmCrc32:
return fields.CRC32
case types.ChecksumAlgorithmCrc32c:
return fields.CRC32C
case types.ChecksumAlgorithmSha1:
return fields.SHA1
case types.ChecksumAlgorithmSha256:
return fields.SHA256
case types.ChecksumAlgorithmCrc64nvme:
return fields.CRC64NVME
case types.ChecksumAlgorithmSha512:
return fields.SHA512
case types.ChecksumAlgorithmMd5:
return fields.MD5
case types.ChecksumAlgorithmXxhash64:
return fields.XXHASH64
case types.ChecksumAlgorithmXxhash3:
return fields.XXHASH3
case types.ChecksumAlgorithmXxhash128:
return fields.XXHASH128
default:
return nil
}
}
func getChecksum(algo types.ChecksumAlgorithm, fields checksumFields) *string {
if checksum := selectChecksum(algo, fields); checksum != nil {
return *checksum
}
return nil
}
func setChecksum(algo types.ChecksumAlgorithm, fields checksumFields, checksum *string) {
if selected := selectChecksum(algo, fields); selected != nil {
*selected = checksum
}
}
func getPartChecksum(part types.Part, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &part.ChecksumCRC32,
CRC32C: &part.ChecksumCRC32C,
SHA1: &part.ChecksumSHA1,
SHA256: &part.ChecksumSHA256,
CRC64NVME: &part.ChecksumCRC64NVME,
SHA512: &part.ChecksumSHA512,
MD5: &part.ChecksumMD5,
XXHASH64: &part.ChecksumXXHASH64,
XXHASH3: &part.ChecksumXXHASH3,
XXHASH128: &part.ChecksumXXHASH128,
})
}
func setPartChecksum(part *types.Part, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &part.ChecksumCRC32,
CRC32C: &part.ChecksumCRC32C,
SHA1: &part.ChecksumSHA1,
SHA256: &part.ChecksumSHA256,
CRC64NVME: &part.ChecksumCRC64NVME,
SHA512: &part.ChecksumSHA512,
MD5: &part.ChecksumMD5,
XXHASH64: &part.ChecksumXXHASH64,
XXHASH3: &part.ChecksumXXHASH3,
XXHASH128: &part.ChecksumXXHASH128,
}, checksum)
}
func getCompletedPartChecksum(part types.CompletedPart, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &part.ChecksumCRC32,
CRC32C: &part.ChecksumCRC32C,
SHA1: &part.ChecksumSHA1,
SHA256: &part.ChecksumSHA256,
CRC64NVME: &part.ChecksumCRC64NVME,
SHA512: &part.ChecksumSHA512,
MD5: &part.ChecksumMD5,
XXHASH64: &part.ChecksumXXHASH64,
XXHASH3: &part.ChecksumXXHASH3,
XXHASH128: &part.ChecksumXXHASH128,
})
}
func completedPartFromPart(part types.Part) types.CompletedPart {
return types.CompletedPart{
ETag: part.ETag,
PartNumber: part.PartNumber,
ChecksumCRC32: part.ChecksumCRC32,
ChecksumCRC32C: part.ChecksumCRC32C,
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumMD5: part.ChecksumMD5,
ChecksumXXHASH64: part.ChecksumXXHASH64,
ChecksumXXHASH3: part.ChecksumXXHASH3,
ChecksumXXHASH128: part.ChecksumXXHASH128,
}
}
func getPutObjectChecksum(out *s3.PutObjectOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func setPutObjectChecksum(in *s3.PutObjectInput, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &in.ChecksumCRC32,
CRC32C: &in.ChecksumCRC32C,
SHA1: &in.ChecksumSHA1,
SHA256: &in.ChecksumSHA256,
CRC64NVME: &in.ChecksumCRC64NVME,
SHA512: &in.ChecksumSHA512,
MD5: &in.ChecksumMD5,
XXHASH64: &in.ChecksumXXHASH64,
XXHASH3: &in.ChecksumXXHASH3,
XXHASH128: &in.ChecksumXXHASH128,
}, checksum)
}
func getGetObjectChecksum(out *s3.GetObjectOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func getHeadObjectChecksum(out *s3.HeadObjectOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func getObjectAttributesChecksum(out *types.Checksum, algo types.ChecksumAlgorithm) *string {
if out == nil {
return nil
}
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func getUploadPartChecksum(out *s3.UploadPartOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func setUploadPartChecksum(in *s3.UploadPartInput, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &in.ChecksumCRC32,
CRC32C: &in.ChecksumCRC32C,
SHA1: &in.ChecksumSHA1,
SHA256: &in.ChecksumSHA256,
CRC64NVME: &in.ChecksumCRC64NVME,
SHA512: &in.ChecksumSHA512,
MD5: &in.ChecksumMD5,
XXHASH64: &in.ChecksumXXHASH64,
XXHASH3: &in.ChecksumXXHASH3,
XXHASH128: &in.ChecksumXXHASH128,
}, checksum)
}
func getCompleteMultipartUploadChecksum(out *s3.CompleteMultipartUploadOutput, algo types.ChecksumAlgorithm) *string {
return getChecksum(algo, checksumFields{
CRC32: &out.ChecksumCRC32,
CRC32C: &out.ChecksumCRC32C,
SHA1: &out.ChecksumSHA1,
SHA256: &out.ChecksumSHA256,
CRC64NVME: &out.ChecksumCRC64NVME,
SHA512: &out.ChecksumSHA512,
MD5: &out.ChecksumMD5,
XXHASH64: &out.ChecksumXXHASH64,
XXHASH3: &out.ChecksumXXHASH3,
XXHASH128: &out.ChecksumXXHASH128,
})
}
func setCompleteMultipartUploadChecksum(in *s3.CompleteMultipartUploadInput, algo types.ChecksumAlgorithm, checksum *string) {
setChecksum(algo, checksumFields{
CRC32: &in.ChecksumCRC32,
CRC32C: &in.ChecksumCRC32C,
SHA1: &in.ChecksumSHA1,
SHA256: &in.ChecksumSHA256,
CRC64NVME: &in.ChecksumCRC64NVME,
SHA512: &in.ChecksumSHA512,
MD5: &in.ChecksumMD5,
XXHASH64: &in.ChecksumXXHASH64,
XXHASH3: &in.ChecksumXXHASH3,
XXHASH128: &in.ChecksumXXHASH128,
}, checksum)
}
func getCopyObjectChecksum(result *types.CopyObjectResult, algo types.ChecksumAlgorithm) *string {
if result == nil {
return nil
}
return getChecksum(algo, checksumFields{
CRC32: &result.ChecksumCRC32,
CRC32C: &result.ChecksumCRC32C,
SHA1: &result.ChecksumSHA1,
SHA256: &result.ChecksumSHA256,
CRC64NVME: &result.ChecksumCRC64NVME,
SHA512: &result.ChecksumSHA512,
MD5: &result.ChecksumMD5,
XXHASH64: &result.ChecksumXXHASH64,
XXHASH3: &result.ChecksumXXHASH3,
XXHASH128: &result.ChecksumXXHASH128,
})
}
func getUploadPartCopyChecksum(result *types.CopyPartResult, algo types.ChecksumAlgorithm) *string {
if result == nil {
return nil
}
return getChecksum(algo, checksumFields{
CRC32: &result.ChecksumCRC32,
CRC32C: &result.ChecksumCRC32C,
SHA1: &result.ChecksumSHA1,
SHA256: &result.ChecksumSHA256,
CRC64NVME: &result.ChecksumCRC64NVME,
SHA512: &result.ChecksumSHA512,
MD5: &result.ChecksumMD5,
XXHASH64: &result.ChecksumXXHASH64,
XXHASH3: &result.ChecksumXXHASH3,
XXHASH128: &result.ChecksumXXHASH128,
})
}
// mp1 needs to be the response from the server
// mp2 needs to be the expected values
// The keys from the server are always converted to lowercase
@@ -921,7 +1272,8 @@ func compareDelObjects(list1, list2 []types.DeletedObject) bool {
func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) {
partSize := size / partCount
var hash hash.Hash
var objHasher hash.Hash
var partHasher hash.Hash
cfg := new(mpCfg)
for _, opt := range opts {
@@ -930,17 +1282,34 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
hash = crc32.NewIEEE()
objHasher = crc32.NewIEEE()
case types.ChecksumAlgorithmCrc32c:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
objHasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case types.ChecksumAlgorithmMd5:
objHasher = md5.New()
case types.ChecksumAlgorithmSha1:
hash = sha1.New()
objHasher = sha1.New()
case types.ChecksumAlgorithmSha256:
hash = sha256.New()
objHasher = sha256.New()
case types.ChecksumAlgorithmSha512:
objHasher = sha512.New()
case types.ChecksumAlgorithmCrc64nvme:
hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
objHasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
case types.ChecksumAlgorithmXxhash64:
objHasher = xxhash.New()
case types.ChecksumAlgorithmXxhash3:
objHasher = xxh3.New()
case types.ChecksumAlgorithmXxhash128:
objHasher = xxh3.New128()
default:
hash = sha256.New()
objHasher = sha256.New()
}
if cfg.checksumAlgorithm != "" {
partHasher, err = NewHasher(cfg.checksumAlgorithm)
if err != nil {
return nil, "", err
}
}
for partNumber := int64(1); partNumber <= partCount; partNumber++ {
@@ -952,18 +1321,28 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
partBuffer := make([]byte, partEnd-partStart+1)
rand.Read(partBuffer)
hash.Write(partBuffer)
objHasher.Write(partBuffer)
if partHasher != nil {
partHasher.Write(partBuffer)
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
pn := int32(partNumber)
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
input := &s3.UploadPartInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: &pn,
ChecksumAlgorithm: cfg.checksumAlgorithm,
})
}
if partHasher != nil {
partChecksum := base64.StdEncoding.EncodeToString(partHasher.Sum(nil))
setUploadPartChecksum(input, cfg.checksumAlgorithm, &partChecksum)
partHasher.Reset()
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := client.UploadPart(ctx, input)
cancel()
if err != nil {
return parts, "", err
@@ -975,22 +1354,11 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
Size: &partSize,
}
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
part.ChecksumCRC32 = out.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
part.ChecksumCRC32C = out.ChecksumCRC32C
case types.ChecksumAlgorithmSha1:
part.ChecksumSHA1 = out.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
part.ChecksumSHA256 = out.ChecksumSHA256
case types.ChecksumAlgorithmCrc64nvme:
part.ChecksumCRC64NVME = out.ChecksumCRC64NVME
}
setPartChecksum(&part, cfg.checksumAlgorithm, getUploadPartChecksum(out, cfg.checksumAlgorithm))
parts = append(parts, part)
}
sum := hash.Sum(nil)
sum := objHasher.Sum(nil)
if cfg.checksumAlgorithm == "" {
csum = hex.EncodeToString(sum[:])
@@ -1277,6 +1645,26 @@ func createObjVersions(client *s3.Client, bucket, object string, count int, opts
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmSha256,
}
case r.res.ChecksumSHA512 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmSha512,
}
case r.res.ChecksumMD5 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmMd5,
}
case r.res.ChecksumXXHASH64 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmXxhash64,
}
case r.res.ChecksumXXHASH3 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmXxhash3,
}
case r.res.ChecksumXXHASH128 != nil:
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
types.ChecksumAlgorithmXxhash128,
}
}
versions = append(versions, version)
@@ -2012,14 +2400,26 @@ func lockObject(client *s3.Client, mode objectLockMode, bucket, object, versionI
func NewHasher(algo types.ChecksumAlgorithm) (hash.Hash, error) {
var hasher hash.Hash
switch algo {
case types.ChecksumAlgorithmMd5:
hasher = md5.New()
case types.ChecksumAlgorithmSha256:
hasher = sha256.New()
case types.ChecksumAlgorithmSha512:
hasher = sha512.New()
case types.ChecksumAlgorithmSha1:
hasher = sha1.New()
case types.ChecksumAlgorithmCrc32:
hasher = crc32.NewIEEE()
case types.ChecksumAlgorithmCrc32c:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case types.ChecksumAlgorithmCrc64nvme:
hasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
case types.ChecksumAlgorithmXxhash64:
hasher = xxhash.New()
case types.ChecksumAlgorithmXxhash3:
hasher = xxh3.New()
case types.ChecksumAlgorithmXxhash128:
hasher = xxh3.New128()
default:
return nil, fmt.Errorf("unsupported hash algorithm: %s", algo)
}
@@ -2027,6 +2427,28 @@ func NewHasher(algo types.ChecksumAlgorithm) (hash.Hash, error) {
return hasher, nil
}
func wrongChecksumForAlgorithm(algo types.ChecksumAlgorithm) (string, error) {
var size int
switch algo {
case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c:
size = 4
case types.ChecksumAlgorithmCrc64nvme, types.ChecksumAlgorithmXxhash64, types.ChecksumAlgorithmXxhash3:
size = 8
case types.ChecksumAlgorithmMd5, types.ChecksumAlgorithmXxhash128:
size = 16
case types.ChecksumAlgorithmSha1:
size = 20
case types.ChecksumAlgorithmSha256:
size = 32
case types.ChecksumAlgorithmSha512:
size = 64
default:
return "", fmt.Errorf("unsupported hash algorithm: %s", algo)
}
return base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0xff}, size)), nil
}
func processCompositeChecksum(hasher hash.Hash, checksum string) error {
data, err := base64.StdEncoding.DecodeString(checksum)
if err != nil {