mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-21 06:36:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums

Browse Source 
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
niksis02
2026-05-04 08:50:39 -07:00
committed by Ben McClelland
parent fbe2a4ba10
commit d2fa265fb8
39 changed files with 1811 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3api/utils/utils_test.go
+211
View File
@@ -707,6 +707,41 @@ func TestIsChecksumAlgorithmValid(t *testing.T) {
},
wantErr: false,
},
{
name: "sha512",
args: args{
alg: types.ChecksumAlgorithmSha512,
},
wantErr: false,
},
{
name: "md5",
args: args{
alg: types.ChecksumAlgorithmMd5,
},
wantErr: false,
},
{
name: "xxhash64",
args: args{
alg: types.ChecksumAlgorithmXxhash64,
},
wantErr: false,
},
{
name: "xxhash3",
args: args{
alg: types.ChecksumAlgorithmXxhash3,
},
wantErr: false,
},
{
name: "xxhash128",
args: args{
alg: types.ChecksumAlgorithmXxhash128,
},
wantErr: false,
},
{
name: "invalid",
args: args{
@@ -806,6 +841,102 @@ func TestIsValidChecksum(t *testing.T) {
},
want: true,
},
{
name: "invalid-crc64nvme",
args: args{
checksum: "ww2FVQ==",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: false,
},
{
name: "valid-crc64nvme",
args: args{
checksum: "AAAAAAAAAAA=",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: true,
},
{
name: "invalid-sha512",
args: args{
checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
algorithm: types.ChecksumAlgorithmSha512,
},
want: false,
},
{
name: "valid-sha512",
args: args{
checksum: "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==",
algorithm: types.ChecksumAlgorithmSha512,
},
want: true,
},
{
name: "invalid-md5",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmMd5,
},
want: false,
},
{
name: "valid-md5",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmMd5,
},
want: true,
},
{
name: "invalid-xxhash64",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: false,
},
{
name: "valid-xxhash64",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: true,
},
{
name: "invalid-xxhash3",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: false,
},
{
name: "valid-xxhash3",
args: args{
checksum: "LQaABTjTlMI=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: true,
},
{
name: "invalid-xxhash128",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: false,
},
{
name: "valid-xxhash128",
args: args{
checksum: "maoG0wFHmNhgAcMkRo1Jfw==",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -1090,6 +1221,86 @@ func Test_checkChecksumTypeAndAlgo(t *testing.T) {
},
wantErr: true,
},
{
name: "full_object-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "composite-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-empty",
args: args{