mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-09-28 10:46:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums

Browse Source 
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
niksis02
2026-05-04 08:50:39 -07:00
committed by Ben McClelland
parent fbe2a4ba10
commit d2fa265fb8
39 changed files with 1811 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3api/controllers/object-put_test.go
+20
View File
@@ -535,6 +535,11 @@ func TestS3ApiController_UploadPart(t *testing.T) {
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
@@ -1214,6 +1219,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-checksum-type": nil,
"x-amz-version-id": emptyStringPtr,
"x-amz-object-size": nil,
@@ -1255,6 +1265,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
ChecksumSHA1: utils.GetStringPtr("sha1"),
ChecksumSHA256: utils.GetStringPtr("sha256"),
ChecksumCRC64NVME: utils.GetStringPtr("crc64nvme"),
ChecksumSHA512: utils.GetStringPtr("sha512"),
ChecksumMD5: utils.GetStringPtr("md5"),
ChecksumXXHASH64: utils.GetStringPtr("xxhash64"),
ChecksumXXHASH3: utils.GetStringPtr("xxhash3"),
ChecksumXXHASH128: utils.GetStringPtr("xxhash128"),
ChecksumType: types.ChecksumTypeComposite,
VersionID: "versionId",
Size: &objSize,
@@ -1269,6 +1284,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
"x-amz-checksum-crc64nvme": utils.GetStringPtr("crc64nvme"),
"x-amz-checksum-sha1": utils.GetStringPtr("sha1"),
"x-amz-checksum-sha256": utils.GetStringPtr("sha256"),
"x-amz-checksum-sha512": utils.GetStringPtr("sha512"),
"x-amz-checksum-md5": utils.GetStringPtr("md5"),
"x-amz-checksum-xxhash64": utils.GetStringPtr("xxhash64"),
"x-amz-checksum-xxhash3": utils.GetStringPtr("xxhash3"),
"x-amz-checksum-xxhash128": utils.GetStringPtr("xxhash128"),
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
"x-amz-version-id": utils.GetStringPtr("versionId"),
"x-amz-object-size": utils.ConvertToStringPtr(objSize),