mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-18 22:14:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums

Browse Source 
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
niksis02
2026-05-04 08:50:39 -07:00
committed by Ben McClelland
parent fbe2a4ba10
commit d2fa265fb8
39 changed files with 1811 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/s3proxy/s3.go
+75
View File
@@ -483,6 +483,21 @@ func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
input.ChecksumSHA256 = nil
}
if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" {
input.ChecksumSHA512 = nil
}
if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" {
input.ChecksumMD5 = nil
}
if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" {
input.ChecksumXXHASH64 = nil
}
if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" {
input.ChecksumXXHASH3 = nil
}
if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" {
input.ChecksumXXHASH128 = nil
}
if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" {
input.ExpectedBucketOwner = nil
}
@@ -518,6 +533,11 @@ func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
ChecksumCRC64NVME: out.ChecksumCRC64NVME,
ChecksumSHA1: out.ChecksumSHA1,
ChecksumSHA256: out.ChecksumSHA256,
ChecksumSHA512: out.ChecksumSHA512,
ChecksumMD5: out.ChecksumMD5,
ChecksumXXHASH64: out.ChecksumXXHASH64,
ChecksumXXHASH3: out.ChecksumXXHASH3,
ChecksumXXHASH128: out.ChecksumXXHASH128,
ChecksumType: &out.ChecksumType,
}
if out.VersionId != nil {
@@ -653,6 +673,11 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
ChecksumCRC64NVME: p.ChecksumCRC64NVME,
ChecksumSHA1: p.ChecksumSHA1,
ChecksumSHA256: p.ChecksumSHA256,
ChecksumSHA512: p.ChecksumSHA512,
ChecksumMD5: p.ChecksumMD5,
ChecksumXXHASH64: p.ChecksumXXHASH64,
ChecksumXXHASH3: p.ChecksumXXHASH3,
ChecksumXXHASH128: p.ChecksumXXHASH128,
})
}
pnm, err := strconv.Atoi(*output.PartNumberMarker)
@@ -709,6 +734,21 @@ func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
input.ChecksumSHA256 = nil
}
if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" {
input.ChecksumSHA512 = nil
}
if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" {
input.ChecksumMD5 = nil
}
if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" {
input.ChecksumXXHASH64 = nil
}
if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" {
input.ChecksumXXHASH3 = nil
}
if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" {
input.ChecksumXXHASH128 = nil
}
if input.ContentMD5 != nil && *input.ContentMD5 == "" {
input.ContentMD5 = nil
}
@@ -790,6 +830,11 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME,
ChecksumSHA1: output.CopyPartResult.ChecksumSHA1,
ChecksumSHA256: output.CopyPartResult.ChecksumSHA256,
ChecksumSHA512: output.CopyPartResult.ChecksumSHA512,
ChecksumMD5: output.CopyPartResult.ChecksumMD5,
ChecksumXXHASH64: output.CopyPartResult.ChecksumXXHASH64,
ChecksumXXHASH3: output.CopyPartResult.ChecksumXXHASH3,
ChecksumXXHASH128: output.CopyPartResult.ChecksumXXHASH128,
}, nil
}
@@ -815,6 +860,21 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" {
input.ChecksumSHA256 = nil
}
if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" {
input.ChecksumSHA512 = nil
}
if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" {
input.ChecksumMD5 = nil
}
if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" {
input.ChecksumXXHASH64 = nil
}
if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" {
input.ChecksumXXHASH3 = nil
}
if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" {
input.ChecksumXXHASH128 = nil
}
if input.ContentDisposition != nil && *input.ContentDisposition == "" {
input.ContentDisposition = nil
}
@@ -913,6 +973,11 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
ChecksumSHA1: input.ChecksumSHA1,
ChecksumSHA256: input.ChecksumSHA256,
ChecksumCRC64NVME: input.ChecksumCRC64NVME,
ChecksumSHA512: input.ChecksumSHA512,
ChecksumMD5: input.ChecksumMD5,
ChecksumXXHASH64: input.ChecksumXXHASH64,
ChecksumXXHASH3: input.ChecksumXXHASH3,
ChecksumXXHASH128: input.ChecksumXXHASH128,
ContentMD5: input.ContentMD5,
ExpectedBucketOwner: input.ExpectedBucketOwner,
GrantFullControl: input.GrantFullControl,
@@ -947,6 +1012,11 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput
ChecksumCRC64NVME: output.ChecksumCRC64NVME,
ChecksumSHA1: output.ChecksumSHA1,
ChecksumSHA256: output.ChecksumSHA256,
ChecksumSHA512: output.ChecksumSHA512,
ChecksumMD5: output.ChecksumMD5,
ChecksumXXHASH64: output.ChecksumXXHASH64,
ChecksumXXHASH3: output.ChecksumXXHASH3,
ChecksumXXHASH128: output.ChecksumXXHASH128,
Size: output.Size,
}, nil
}
@@ -1292,6 +1362,11 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input s3response.CopyObjectInp
ChecksumCRC64NVME: out.CopyObjectResult.ChecksumCRC64NVME,
ChecksumSHA1: out.CopyObjectResult.ChecksumSHA1,
ChecksumSHA256: out.CopyObjectResult.ChecksumSHA256,
ChecksumSHA512: out.CopyObjectResult.ChecksumSHA512,
ChecksumMD5: out.CopyObjectResult.ChecksumMD5,
ChecksumXXHASH64: out.CopyObjectResult.ChecksumXXHASH64,
ChecksumXXHASH3: out.CopyObjectResult.ChecksumXXHASH3,
ChecksumXXHASH128: out.CopyObjectResult.ChecksumXXHASH128,
ChecksumType: out.CopyObjectResult.ChecksumType,
ETag: out.CopyObjectResult.ETag,
LastModified: out.CopyObjectResult.LastModified,