mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-20 06:54:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums

Browse Source 
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
niksis02
2026-05-04 08:50:39 -07:00
committed by Ben McClelland
parent fbe2a4ba10
commit d2fa265fb8
39 changed files with 1811 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/GetObject.go
+45 -131
View File
@@ -233,38 +233,13 @@ func GetObject_invalid_parent(s *S3Conf) error {
func GetObject_checksums(s *S3Conf) error {
testName := "GetObject_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []struct {
key string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-4",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{
key: "obj-5",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
}
for i, el := range objs {
out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{
Bucket: &bucket,
Key: &el.key,
ChecksumAlgorithm: el.checksumAlgo,
}, s3client)
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
@@ -272,7 +247,7 @@ func GetObject_checksums(s *S3Conf) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &el.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -281,28 +256,11 @@ func GetObject_checksums(s *S3Conf) error {
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v",
el.key, types.ChecksumTypeFullObject, res.ChecksumType)
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v",
key, types.ChecksumTypeFullObject, res.ChecksumType)
}
if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) {
return fmt.Errorf("expected crc32 checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32))
}
if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) {
return fmt.Errorf("expected crc32c checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C))
}
if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) {
return fmt.Errorf("expected sha1 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1))
}
if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum to be %v, instead got %v",
getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256))
}
if getString(res.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) {
return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v",
getString(out.res.ChecksumCRC64NVME), getString(res.ChecksumCRC64NVME))
if got, expected := getString(getGetObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got)
}
}
@@ -313,43 +271,31 @@ func GetObject_checksums(s *S3Conf) error {
func GetObject_dir_object_checksum(s *S3Conf) error {
testName := "GetObject_dir_object_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
for i, obj := range []struct {
key string
expectedSum string
for i, test := range []struct {
checksumAlgo types.ChecksumAlgorithm
expectedSum string
}{
{
key: "obj-1/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3/",
expectedSum: "AAAAAAAAAAA=",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
{
key: "obj-4/",
expectedSum: "2jmj7l5rSw0yVb/vlWAYkK/YBwk=",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-5/",
expectedSum: "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
{types.ChecksumAlgorithmCrc32, "AAAAAA=="},
{types.ChecksumAlgorithmCrc32c, "AAAAAA=="},
{types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="},
{types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="},
{types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},
{types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="},
{types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="},
{types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="},
{types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="},
{types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, &s3.PutObjectInput{
key := fmt.Sprintf("obj-%v/", i+1)
input := &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj.key,
ChecksumAlgorithm: obj.checksumAlgo,
})
Key: &key,
ChecksumAlgorithm: test.checksumAlgo,
}
setPutObjectChecksum(input, test.checksumAlgo, &test.expectedSum)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, input)
cancel()
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
@@ -358,7 +304,7 @@ func GetObject_dir_object_checksum(s *S3Conf) error {
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &obj.key,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
@@ -368,25 +314,13 @@ func GetObject_dir_object_checksum(s *S3Conf) error {
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("test %v failed: expected the %v object checksum type to be %v, instaed got %v",
i+1, obj.key, types.ChecksumTypeFullObject, res.ChecksumType)
i+1, key, types.ChecksumTypeFullObject, res.ChecksumType)
}
var gotSum *string
switch obj.checksumAlgo {
case types.ChecksumAlgorithmCrc32:
gotSum = res.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
gotSum = res.ChecksumCRC32C
case types.ChecksumAlgorithmCrc64nvme:
gotSum = res.ChecksumCRC64NVME
case types.ChecksumAlgorithmSha1:
gotSum = res.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
gotSum = res.ChecksumSHA256
}
gotSum := getGetObjectChecksum(res, test.checksumAlgo)
if getString(gotSum) != obj.expectedSum {
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, obj.checksumAlgo, obj.expectedSum, getString(gotSum))
if getString(gotSum) != test.expectedSum {
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, test.checksumAlgo, test.expectedSum, getString(gotSum))
}
}
@@ -885,20 +819,10 @@ func GetObject_not_enabled_checksum_mode(s *S3Conf) error {
return err
}
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getGetObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
@@ -1704,20 +1628,10 @@ func GetObject_ranged_with_checksum_mode(s *S3Conf) error {
testName := "GetObject_ranged_with_checksum_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
checkNoChecksums := func(res *s3.GetObjectOutput) error {
if res.ChecksumCRC32 != nil {
return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32)
}
if res.ChecksumCRC32C != nil {
return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C)
}
if res.ChecksumSHA1 != nil {
return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1)
}
if res.ChecksumSHA256 != nil {
return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256)
}
if res.ChecksumCRC64NVME != nil {
return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME)
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getGetObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
}