mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-18 21:26:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums

Browse Source 
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
niksis02
2026-05-04 08:50:39 -07:00
committed by Ben McClelland
parent fbe2a4ba10
commit d2fa265fb8
39 changed files with 1811 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3api/utils/chunk-reader.go
+11 -1
View File
@@ -79,6 +79,11 @@ const (
checksumTypeSha1 checksumType = "x-amz-checksum-sha1"
checksumTypeSha256 checksumType = "x-amz-checksum-sha256"
checksumTypeCrc64nvme checksumType = "x-amz-checksum-crc64nvme"
checksumTypeSha512 checksumType = "x-amz-checksum-sha512"
checksumTypeMd5 checksumType = "x-amz-checksum-md5"
checksumTypeXxhash64 checksumType = "x-amz-checksum-xxhash64"
checksumTypeXxhash3 checksumType = "x-amz-checksum-xxhash3"
checksumTypeXxhash128 checksumType = "x-amz-checksum-xxhash128"
)
func (c checksumType) isValid() bool {
@@ -86,7 +91,12 @@ func (c checksumType) isValid() bool {
c == checksumTypeCrc32c ||
c == checksumTypeSha1 ||
c == checksumTypeSha256 ||
c == checksumTypeCrc64nvme
c == checksumTypeCrc64nvme ||
c == checksumTypeSha512 ||
c == checksumTypeMd5 ||
c == checksumTypeXxhash64 ||
c == checksumTypeXxhash3 ||
c == checksumTypeXxhash128
}
// Extracts and validates the checksum type from the 'X-Amz-Trailer' header
s3api/utils/csum-reader.go
+75 -6
View File
@@ -18,6 +18,7 @@ import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"errors"
@@ -28,14 +29,18 @@ import (
"io"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
)
// HashType identifies the checksum algorithm to be used
type HashType string
const (
// HashTypeMd5 generates MD5 checksum for the data stream
// HashTypeContentMD5 generates MD5 checksum for the Content-MD5 header.
HashTypeContentMD5 HashType = "content-md5"
// HashTypeMd5 generates MD5 Base64-Encoded checksum for the data stream
HashTypeMd5 HashType = "md5"
// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
HashTypeSha256 HashType = "sha256"
@@ -43,12 +48,20 @@ const (
HashTypeSha256Hex HashType = "sha256-hex"
// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
HashTypeSha1 HashType = "sha1"
// HashTypeSha512 generates SHA512 Base64-Encoded checksum for the data stream
HashTypeSha512 HashType = "sha512"
// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
HashTypeCRC32 HashType = "crc32"
// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
HashTypeCRC32C HashType = "crc32c"
// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
HashTypeCRC64NVME HashType = "crc64nvme"
// HashTypeXXHASH64 generates XXHASH64 Base64-Encoded checksum for the data stream
HashTypeXXHASH64 HashType = "xxhash64"
// HashTypeXXHASH3 generates XXHASH3 Base64-Encoded checksum for the data stream
HashTypeXXHASH3 HashType = "xxhash3"
// HashTypeXXHASH128 generates XXHASH128 Base64-Encoded checksum for the data stream
HashTypeXXHASH128 HashType = "xxhash128"
// HashTypeNone is a no-op checksum for the data stream
HashTypeNone HashType = "none"
)
@@ -75,7 +88,7 @@ var (
func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) {
var hash hash.Hash
switch ht {
case HashTypeMd5:
case HashTypeContentMD5, HashTypeMd5:
hash = md5.New()
case HashTypeSha256Hex:
hash = sha256.New()
@@ -83,12 +96,20 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
hash = sha256.New()
case HashTypeSha1:
hash = sha1.New()
case HashTypeSha512:
hash = sha512.New()
case HashTypeCRC32:
hash = crc32.NewIEEE()
case HashTypeCRC32C:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeCRC64NVME:
hash = crc64.New(crc64NVMETable)
case HashTypeXXHASH64:
hash = xxhash.New()
case HashTypeXXHASH3:
hash = xxh3.New()
case HashTypeXXHASH128:
hash = xxh3.New128()
case HashTypeNone:
hash = noop{}
default:
@@ -112,11 +133,16 @@ func (hr *HashReader) Read(p []byte) (int, error) {
}
if errors.Is(readerr, io.EOF) && hr.sum != "" {
switch hr.hashType {
case HashTypeMd5:
case HashTypeContentMD5:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrBadDigest)
}
case HashTypeMd5:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmMd5)
}
case HashTypeSha256Hex:
sum := hr.Sum()
if sum != hr.sum {
@@ -142,11 +168,31 @@ func (hr *HashReader) Read(p []byte) (int, error) {
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
}
case HashTypeSha512:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha512)
}
case HashTypeCRC64NVME:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
}
case HashTypeXXHASH64:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash64)
}
case HashTypeXXHASH3:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash3)
}
case HashTypeXXHASH128:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash128)
}
default:
return n, errInvalidHashType
}
@@ -161,7 +207,7 @@ func (hr *HashReader) SetReader(r io.Reader) {
// Sum returns the checksum hash of the data read so far
func (hr *HashReader) Sum() string {
switch hr.hashType {
case HashTypeMd5:
case HashTypeContentMD5, HashTypeMd5:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256Hex:
return hex.EncodeToString(hr.hash.Sum(nil))
@@ -173,8 +219,16 @@ func (hr *HashReader) Sum() string {
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha512:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeCRC64NVME:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH64:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH3:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH128:
return Base64SumString(hr.hash.Sum(nil))
default:
return ""
}
@@ -197,8 +251,8 @@ func (n noop) Reset() {}
func (n noop) Size() int { return 0 }
func (n noop) BlockSize() int { return 1 }
// IsChecksumComposable tests if the final foll object crc can be calculated
// based on the part crc values.
// IsChecksumComposable tests if the final full-object crc can be calculated
// from the part crc values.
func IsChecksumComposable(algo types.ChecksumAlgorithm) bool {
switch algo {
case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c, types.ChecksumAlgorithmCrc64nvme:
@@ -309,17 +363,32 @@ func AddCRCChecksum(algo types.ChecksumAlgorithm, crc, partCrc string, partLen i
// - CRC32C
// - SHA1
// - SHA256
// - SHA512
// - MD5
// - XXHASH64
// - XXHASH3
// - XXHASH128
func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
var hasher hash.Hash
switch ht {
case HashTypeMd5:
hasher = md5.New()
case HashTypeSha256:
hasher = sha256.New()
case HashTypeSha1:
hasher = sha1.New()
case HashTypeSha512:
hasher = sha512.New()
case HashTypeCRC32:
hasher = crc32.NewIEEE()
case HashTypeCRC32C:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeXXHASH64:
hasher = xxhash.New()
case HashTypeXXHASH3:
hasher = xxh3.New()
case HashTypeXXHASH128:
hasher = xxh3.New128()
case HashTypeNone:
hasher = noop{}
default:
s3api/utils/csum-reader_test.go
+99
View File
@@ -15,12 +15,22 @@
package utils
import (
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/stretchr/testify/assert"
"github.com/zeebo/xxh3"
)
func TestAddCRCChecksum_CRC32(t *testing.T) {
@@ -118,3 +128,92 @@ func TestAddCRCChecksum_CRC64NVME(t *testing.T) {
t.Errorf("CRC64NVME combine mismatch: got %x, want %x", combinedVal, crcFull)
}
}
func base64HashForTest(h hash.Hash, data []byte) string {
h.Write(data)
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func TestNewHashReader_NewChecksumAlgorithms(t *testing.T) {
data := []byte("checksum payload")
tests := []struct {
name string
hashType HashType
hasher hash.Hash
}{
{name: "md5", hashType: HashTypeMd5, hasher: md5.New()},
{name: "sha1", hashType: HashTypeSha1, hasher: sha1.New()},
{name: "sha256", hashType: HashTypeSha256, hasher: sha256.New()},
{name: "sha512", hashType: HashTypeSha512, hasher: sha512.New()},
{name: "crc32", hashType: HashTypeCRC32, hasher: crc32.NewIEEE()},
{name: "crc32c", hashType: HashTypeCRC32C, hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
{name: "crc64nvme", hashType: HashTypeCRC64NVME, hasher: crc64.New(crc64NVMETable)},
{name: "xxhash64", hashType: HashTypeXXHASH64, hasher: xxhash.New()},
{name: "xxhash3", hashType: HashTypeXXHASH3, hasher: xxh3.New()},
{name: "xxhash128", hashType: HashTypeXXHASH128, hasher: xxh3.New128()},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
expected := base64HashForTest(tt.hasher, data)
rdr, err := NewHashReader(bytes.NewReader(data), expected, tt.hashType)
if !assert.NoError(t, err) {
return
}
_, err = io.Copy(io.Discard, rdr)
assert.NoError(t, err)
assert.Equal(t, expected, rdr.Sum())
})
}
}
func TestNewCompositeChecksumReader_NewChecksumAlgorithms(t *testing.T) {
part1 := []byte("part one")
part2 := []byte("part two")
tests := []struct {
name string
hashType HashType
newHasher func() hash.Hash
}{
{name: "md5", hashType: HashTypeMd5, newHasher: md5.New},
{name: "sha1", hashType: HashTypeSha1, newHasher: sha1.New},
{name: "sha256", hashType: HashTypeSha256, newHasher: sha256.New},
{name: "sha512", hashType: HashTypeSha512, newHasher: sha512.New},
{name: "crc32", hashType: HashTypeCRC32, newHasher: func() hash.Hash { return crc32.NewIEEE() }},
{name: "crc32c", hashType: HashTypeCRC32C, newHasher: func() hash.Hash { return crc32.New(crc32.MakeTable(crc32.Castagnoli)) }},
{name: "xxhash64", hashType: HashTypeXXHASH64, newHasher: func() hash.Hash { return xxhash.New() }},
{name: "xxhash3", hashType: HashTypeXXHASH3, newHasher: func() hash.Hash { return xxh3.New() }},
{name: "xxhash128", hashType: HashTypeXXHASH128, newHasher: func() hash.Hash { return xxh3.New128() }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
part1Sum := base64HashForTest(tt.newHasher(), part1)
part2Sum := base64HashForTest(tt.newHasher(), part2)
composite, err := NewCompositeChecksumReader(tt.hashType)
if !assert.NoError(t, err) {
return
}
assert.NoError(t, composite.Process(part1Sum))
assert.NoError(t, composite.Process(part2Sum))
part1Bytes, err := base64.StdEncoding.DecodeString(part1Sum)
if !assert.NoError(t, err) {
return
}
part2Bytes, err := base64.StdEncoding.DecodeString(part2Sum)
if !assert.NoError(t, err) {
return
}
expectedHasher := tt.newHasher()
expectedHasher.Write(part1Bytes)
expectedHasher.Write(part2Bytes)
expected := base64.StdEncoding.EncodeToString(expectedHasher.Sum(nil))
assert.Equal(t, expected, composite.Sum())
})
}
}
s3api/utils/unsigned-chunk-reader.go
+14
View File
@@ -17,8 +17,10 @@ package utils
import (
"bufio"
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"errors"
"hash"
@@ -30,8 +32,10 @@ import (
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
)
var (
@@ -353,6 +357,16 @@ func getHasher(ct checksumType) (hash.Hash, error) {
return sha1.New(), nil
case checksumTypeSha256:
return sha256.New(), nil
case checksumTypeSha512:
return sha512.New(), nil
case checksumTypeMd5:
return md5.New(), nil
case checksumTypeXxhash64:
return xxhash.New(), nil
case checksumTypeXxhash3:
return xxh3.New(), nil
case checksumTypeXxhash128:
return xxh3.New128(), nil
default:
return nil, errors.New("unsupported checksum type")
}
s3api/utils/utils.go
+31 -1
View File
@@ -721,8 +721,13 @@ var checksumLengths = map[types.ChecksumAlgorithm]int{
types.ChecksumAlgorithmCrc32: 4,
types.ChecksumAlgorithmCrc32c: 4,
types.ChecksumAlgorithmCrc64nvme: 8,
types.ChecksumAlgorithmMd5: 16,
types.ChecksumAlgorithmSha1: 20,
types.ChecksumAlgorithmSha256: 32,
types.ChecksumAlgorithmSha512: 64,
types.ChecksumAlgorithmXxhash64: 8,
types.ChecksumAlgorithmXxhash3: 8,
types.ChecksumAlgorithmXxhash128: 16,
}
func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
@@ -753,7 +758,12 @@ func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
alg != types.ChecksumAlgorithmCrc32c &&
alg != types.ChecksumAlgorithmSha1 &&
alg != types.ChecksumAlgorithmSha256 &&
alg != types.ChecksumAlgorithmCrc64nvme {
alg != types.ChecksumAlgorithmCrc64nvme &&
alg != types.ChecksumAlgorithmSha512 &&
alg != types.ChecksumAlgorithmMd5 &&
alg != types.ChecksumAlgorithmXxhash64 &&
alg != types.ChecksumAlgorithmXxhash3 &&
alg != types.ChecksumAlgorithmXxhash128 {
debuglogger.Logf("invalid checksum algorithm: %v\n", alg)
return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
}
@@ -799,6 +809,26 @@ var checksumMap checksumSchema = checksumSchema{
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmSha512: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmMd5: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash64: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash3: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash128: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
// Both could be empty
"": checksumTypeSchema{
"": struct{}{},
s3api/utils/utils_test.go
+211
View File
@@ -707,6 +707,41 @@ func TestIsChecksumAlgorithmValid(t *testing.T) {
},
wantErr: false,
},
{
name: "sha512",
args: args{
alg: types.ChecksumAlgorithmSha512,
},
wantErr: false,
},
{
name: "md5",
args: args{
alg: types.ChecksumAlgorithmMd5,
},
wantErr: false,
},
{
name: "xxhash64",
args: args{
alg: types.ChecksumAlgorithmXxhash64,
},
wantErr: false,
},
{
name: "xxhash3",
args: args{
alg: types.ChecksumAlgorithmXxhash3,
},
wantErr: false,
},
{
name: "xxhash128",
args: args{
alg: types.ChecksumAlgorithmXxhash128,
},
wantErr: false,
},
{
name: "invalid",
args: args{
@@ -806,6 +841,102 @@ func TestIsValidChecksum(t *testing.T) {
},
want: true,
},
{
name: "invalid-crc64nvme",
args: args{
checksum: "ww2FVQ==",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: false,
},
{
name: "valid-crc64nvme",
args: args{
checksum: "AAAAAAAAAAA=",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: true,
},
{
name: "invalid-sha512",
args: args{
checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
algorithm: types.ChecksumAlgorithmSha512,
},
want: false,
},
{
name: "valid-sha512",
args: args{
checksum: "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==",
algorithm: types.ChecksumAlgorithmSha512,
},
want: true,
},
{
name: "invalid-md5",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmMd5,
},
want: false,
},
{
name: "valid-md5",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmMd5,
},
want: true,
},
{
name: "invalid-xxhash64",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: false,
},
{
name: "valid-xxhash64",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: true,
},
{
name: "invalid-xxhash3",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: false,
},
{
name: "valid-xxhash3",
args: args{
checksum: "LQaABTjTlMI=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: true,
},
{
name: "invalid-xxhash128",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: false,
},
{
name: "valid-xxhash128",
args: args{
checksum: "maoG0wFHmNhgAcMkRo1Jfw==",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -1090,6 +1221,86 @@ func Test_checkChecksumTypeAndAlgo(t *testing.T) {
},
wantErr: true,
},
{
name: "full_object-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "composite-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-empty",
args: args{