feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums

Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
niksis02
2026-05-04 08:50:39 -07:00
committed by Ben McClelland
parent fbe2a4ba10
commit d2fa265fb8
39 changed files with 1811 additions and 1025 deletions
+35
View File
@@ -37,6 +37,11 @@ type PutObjectOutput struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
Size *int64
ChecksumType types.ChecksumType
}
@@ -52,6 +57,11 @@ type Part struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
}
func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -375,6 +385,11 @@ type CopyObjectResult struct {
ChecksumCRC64NVME *string
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
ChecksumType types.ChecksumType
ETag *string
LastModified *time.Time
@@ -404,6 +419,11 @@ type CopyPartResult struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
// not included in the body
CopySourceVersionId string `xml:"-"`
@@ -439,6 +459,11 @@ type CompleteMultipartUploadResult struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
ChecksumType *types.ChecksumType
}
@@ -559,6 +584,11 @@ type PutObjectInput struct {
ChecksumSHA1 *string
ChecksumSHA256 *string
ChecksumCRC64NVME *string
ChecksumSHA512 *string
ChecksumMD5 *string
ChecksumXXHASH64 *string
ChecksumXXHASH3 *string
ChecksumXXHASH128 *string
ContentMD5 *string
ExpectedBucketOwner *string
GrantFullControl *string
@@ -731,6 +761,11 @@ type Checksum struct {
SHA1 *string
SHA256 *string
CRC64NVME *string
SHA512 *string
MD5 *string
XXHASH64 *string
XXHASH3 *string
XXHASH128 *string
}
// LocationConstraint represents the GetBucketLocation response