mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-08-17 12:46:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums

Browse Source 
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
niksis02
2026-05-04 08:50:39 -07:00
committed by Ben McClelland
parent fbe2a4ba10
commit d2fa265fb8
39 changed files with 1811 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3api/controllers/bucket-post.go
+10
View File
@@ -222,6 +222,11 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
})
if err != nil {
return &Response{
@@ -285,6 +290,11 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
"x-amz-version-id": utils.GetStringPtr(res.VersionID),
},
s3api/controllers/bucket-post_test.go
+5
View File
@@ -547,6 +547,11 @@ func TestS3ApiController_POSTObject(t *testing.T) {
"x-amz-checksum-crc64nvme": utils.GetStringPtr(""),
"x-amz-checksum-sha1": utils.GetStringPtr(""),
"x-amz-checksum-sha256": utils.GetStringPtr(""),
"x-amz-checksum-sha512": utils.GetStringPtr(""),
"x-amz-checksum-md5": utils.GetStringPtr(""),
"x-amz-checksum-xxhash64": utils.GetStringPtr(""),
"x-amz-checksum-xxhash3": utils.GetStringPtr(""),
"x-amz-checksum-xxhash128": utils.GetStringPtr(""),
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
"x-amz-version-id": utils.GetStringPtr("vid-123"),
},
s3api/controllers/object-get.go
+5
View File
@@ -556,6 +556,11 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"Content-Type": utils.ApplyOverride(res.ContentType, responseOverrides["Content-Type"]),
"x-amz-version-id": res.VersionId,
"Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength),
s3api/controllers/object-get_test.go
+10
View File
@@ -799,6 +799,11 @@ func TestS3ApiController_GetObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
@@ -844,6 +849,11 @@ func TestS3ApiController_GetObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
s3api/controllers/object-head.go
+5
View File
@@ -188,6 +188,11 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-version-id": res.VersionId,
"x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount),
"x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode),
s3api/controllers/object-head_test.go
+10
View File
@@ -189,6 +189,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
@@ -233,6 +238,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
"x-amz-checksum-crc32c": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-version-id": nil,
"x-amz-mp-parts-count": nil,
"x-amz-object-lock-mode": nil,
s3api/controllers/object-post.go
+5
View File
@@ -375,6 +375,11 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
ChecksumType: checksumType,
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
s3api/controllers/object-put.go
+20
View File
@@ -306,6 +306,11 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
})
var headers map[string]*string
if err == nil {
@@ -316,6 +321,11 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
}
}
return &Response{
@@ -789,6 +799,11 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]),
ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]),
ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]),
ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]),
ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]),
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
})
@@ -800,6 +815,11 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"x-amz-checksum-sha512": res.ChecksumSHA512,
"x-amz-checksum-md5": res.ChecksumMD5,
"x-amz-checksum-xxhash64": res.ChecksumXXHASH64,
"x-amz-checksum-xxhash3": res.ChecksumXXHASH3,
"x-amz-checksum-xxhash128": res.ChecksumXXHASH128,
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
"x-amz-version-id": &res.VersionID,
"x-amz-object-size": utils.ConvertPtrToStringPtr(res.Size),
s3api/controllers/object-put_test.go
+20
View File
@@ -535,6 +535,11 @@ func TestS3ApiController_UploadPart(t *testing.T) {
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
},
MetaOpts: &MetaOptions{
BucketOwner: "root",
@@ -1214,6 +1219,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
"x-amz-checksum-crc64nvme": nil,
"x-amz-checksum-sha1": nil,
"x-amz-checksum-sha256": nil,
"x-amz-checksum-sha512": nil,
"x-amz-checksum-md5": nil,
"x-amz-checksum-xxhash64": nil,
"x-amz-checksum-xxhash3": nil,
"x-amz-checksum-xxhash128": nil,
"x-amz-checksum-type": nil,
"x-amz-version-id": emptyStringPtr,
"x-amz-object-size": nil,
@@ -1255,6 +1265,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
ChecksumSHA1: utils.GetStringPtr("sha1"),
ChecksumSHA256: utils.GetStringPtr("sha256"),
ChecksumCRC64NVME: utils.GetStringPtr("crc64nvme"),
ChecksumSHA512: utils.GetStringPtr("sha512"),
ChecksumMD5: utils.GetStringPtr("md5"),
ChecksumXXHASH64: utils.GetStringPtr("xxhash64"),
ChecksumXXHASH3: utils.GetStringPtr("xxhash3"),
ChecksumXXHASH128: utils.GetStringPtr("xxhash128"),
ChecksumType: types.ChecksumTypeComposite,
VersionID: "versionId",
Size: &objSize,
@@ -1269,6 +1284,11 @@ func TestS3ApiController_PutObject(t *testing.T) {
"x-amz-checksum-crc64nvme": utils.GetStringPtr("crc64nvme"),
"x-amz-checksum-sha1": utils.GetStringPtr("sha1"),
"x-amz-checksum-sha256": utils.GetStringPtr("sha256"),
"x-amz-checksum-sha512": utils.GetStringPtr("sha512"),
"x-amz-checksum-md5": utils.GetStringPtr("md5"),
"x-amz-checksum-xxhash64": utils.GetStringPtr("xxhash64"),
"x-amz-checksum-xxhash3": utils.GetStringPtr("xxhash3"),
"x-amz-checksum-xxhash128": utils.GetStringPtr("xxhash128"),
"x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)),
"x-amz-version-id": utils.GetStringPtr("versionId"),
"x-amz-object-size": utils.ConvertToStringPtr(objSize),
s3api/middlewares/checksum.go
+2 -2
View File
@@ -49,7 +49,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
var err error
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
r, err = utils.NewHashReader(r, md5sum, utils.HashTypeMd5)
r, err = utils.NewHashReader(r, md5sum, utils.HashTypeContentMD5)
return r
})
if err != nil {
@@ -70,7 +70,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
return s3err.GetAPIError(s3err.ErrInvalidDigest)
}
rdr, err = utils.NewHashReader(bytes.NewReader(body), md5sum, utils.HashTypeMd5)
rdr, err = utils.NewHashReader(bytes.NewReader(body), md5sum, utils.HashTypeContentMD5)
if err != nil {
return err
}
s3api/utils/chunk-reader.go
+11 -1
View File
@@ -79,6 +79,11 @@ const (
checksumTypeSha1 checksumType = "x-amz-checksum-sha1"
checksumTypeSha256 checksumType = "x-amz-checksum-sha256"
checksumTypeCrc64nvme checksumType = "x-amz-checksum-crc64nvme"
checksumTypeSha512 checksumType = "x-amz-checksum-sha512"
checksumTypeMd5 checksumType = "x-amz-checksum-md5"
checksumTypeXxhash64 checksumType = "x-amz-checksum-xxhash64"
checksumTypeXxhash3 checksumType = "x-amz-checksum-xxhash3"
checksumTypeXxhash128 checksumType = "x-amz-checksum-xxhash128"
)
func (c checksumType) isValid() bool {
@@ -86,7 +91,12 @@ func (c checksumType) isValid() bool {
c == checksumTypeCrc32c ||
c == checksumTypeSha1 ||
c == checksumTypeSha256 ||
c == checksumTypeCrc64nvme
c == checksumTypeCrc64nvme ||
c == checksumTypeSha512 ||
c == checksumTypeMd5 ||
c == checksumTypeXxhash64 ||
c == checksumTypeXxhash3 ||
c == checksumTypeXxhash128
}
// Extracts and validates the checksum type from the 'X-Amz-Trailer' header
s3api/utils/csum-reader.go
+75 -6
View File
@@ -18,6 +18,7 @@ import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"errors"
@@ -28,14 +29,18 @@ import (
"io"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
)
// HashType identifies the checksum algorithm to be used
type HashType string
const (
// HashTypeMd5 generates MD5 checksum for the data stream
// HashTypeContentMD5 generates MD5 checksum for the Content-MD5 header.
HashTypeContentMD5 HashType = "content-md5"
// HashTypeMd5 generates MD5 Base64-Encoded checksum for the data stream
HashTypeMd5 HashType = "md5"
// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
HashTypeSha256 HashType = "sha256"
@@ -43,12 +48,20 @@ const (
HashTypeSha256Hex HashType = "sha256-hex"
// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
HashTypeSha1 HashType = "sha1"
// HashTypeSha512 generates SHA512 Base64-Encoded checksum for the data stream
HashTypeSha512 HashType = "sha512"
// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
HashTypeCRC32 HashType = "crc32"
// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
HashTypeCRC32C HashType = "crc32c"
// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
HashTypeCRC64NVME HashType = "crc64nvme"
// HashTypeXXHASH64 generates XXHASH64 Base64-Encoded checksum for the data stream
HashTypeXXHASH64 HashType = "xxhash64"
// HashTypeXXHASH3 generates XXHASH3 Base64-Encoded checksum for the data stream
HashTypeXXHASH3 HashType = "xxhash3"
// HashTypeXXHASH128 generates XXHASH128 Base64-Encoded checksum for the data stream
HashTypeXXHASH128 HashType = "xxhash128"
// HashTypeNone is a no-op checksum for the data stream
HashTypeNone HashType = "none"
)
@@ -75,7 +88,7 @@ var (
func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) {
var hash hash.Hash
switch ht {
case HashTypeMd5:
case HashTypeContentMD5, HashTypeMd5:
hash = md5.New()
case HashTypeSha256Hex:
hash = sha256.New()
@@ -83,12 +96,20 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
hash = sha256.New()
case HashTypeSha1:
hash = sha1.New()
case HashTypeSha512:
hash = sha512.New()
case HashTypeCRC32:
hash = crc32.NewIEEE()
case HashTypeCRC32C:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeCRC64NVME:
hash = crc64.New(crc64NVMETable)
case HashTypeXXHASH64:
hash = xxhash.New()
case HashTypeXXHASH3:
hash = xxh3.New()
case HashTypeXXHASH128:
hash = xxh3.New128()
case HashTypeNone:
hash = noop{}
default:
@@ -112,11 +133,16 @@ func (hr *HashReader) Read(p []byte) (int, error) {
}
if errors.Is(readerr, io.EOF) && hr.sum != "" {
switch hr.hashType {
case HashTypeMd5:
case HashTypeContentMD5:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrBadDigest)
}
case HashTypeMd5:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmMd5)
}
case HashTypeSha256Hex:
sum := hr.Sum()
if sum != hr.sum {
@@ -142,11 +168,31 @@ func (hr *HashReader) Read(p []byte) (int, error) {
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
}
case HashTypeSha512:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha512)
}
case HashTypeCRC64NVME:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
}
case HashTypeXXHASH64:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash64)
}
case HashTypeXXHASH3:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash3)
}
case HashTypeXXHASH128:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash128)
}
default:
return n, errInvalidHashType
}
@@ -161,7 +207,7 @@ func (hr *HashReader) SetReader(r io.Reader) {
// Sum returns the checksum hash of the data read so far
func (hr *HashReader) Sum() string {
switch hr.hashType {
case HashTypeMd5:
case HashTypeContentMD5, HashTypeMd5:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256Hex:
return hex.EncodeToString(hr.hash.Sum(nil))
@@ -173,8 +219,16 @@ func (hr *HashReader) Sum() string {
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha512:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeCRC64NVME:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH64:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH3:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeXXHASH128:
return Base64SumString(hr.hash.Sum(nil))
default:
return ""
}
@@ -197,8 +251,8 @@ func (n noop) Reset() {}
func (n noop) Size() int { return 0 }
func (n noop) BlockSize() int { return 1 }
// IsChecksumComposable tests if the final foll object crc can be calculated
// based on the part crc values.
// IsChecksumComposable tests if the final full-object crc can be calculated
// from the part crc values.
func IsChecksumComposable(algo types.ChecksumAlgorithm) bool {
switch algo {
case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c, types.ChecksumAlgorithmCrc64nvme:
@@ -309,17 +363,32 @@ func AddCRCChecksum(algo types.ChecksumAlgorithm, crc, partCrc string, partLen i
// - CRC32C
// - SHA1
// - SHA256
// - SHA512
// - MD5
// - XXHASH64
// - XXHASH3
// - XXHASH128
func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
var hasher hash.Hash
switch ht {
case HashTypeMd5:
hasher = md5.New()
case HashTypeSha256:
hasher = sha256.New()
case HashTypeSha1:
hasher = sha1.New()
case HashTypeSha512:
hasher = sha512.New()
case HashTypeCRC32:
hasher = crc32.NewIEEE()
case HashTypeCRC32C:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeXXHASH64:
hasher = xxhash.New()
case HashTypeXXHASH3:
hasher = xxh3.New()
case HashTypeXXHASH128:
hasher = xxh3.New128()
case HashTypeNone:
hasher = noop{}
default:
s3api/utils/csum-reader_test.go
+99
View File
@@ -15,12 +15,22 @@
package utils
import (
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/stretchr/testify/assert"
"github.com/zeebo/xxh3"
)
func TestAddCRCChecksum_CRC32(t *testing.T) {
@@ -118,3 +128,92 @@ func TestAddCRCChecksum_CRC64NVME(t *testing.T) {
t.Errorf("CRC64NVME combine mismatch: got %x, want %x", combinedVal, crcFull)
}
}
func base64HashForTest(h hash.Hash, data []byte) string {
h.Write(data)
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func TestNewHashReader_NewChecksumAlgorithms(t *testing.T) {
data := []byte("checksum payload")
tests := []struct {
name string
hashType HashType
hasher hash.Hash
}{
{name: "md5", hashType: HashTypeMd5, hasher: md5.New()},
{name: "sha1", hashType: HashTypeSha1, hasher: sha1.New()},
{name: "sha256", hashType: HashTypeSha256, hasher: sha256.New()},
{name: "sha512", hashType: HashTypeSha512, hasher: sha512.New()},
{name: "crc32", hashType: HashTypeCRC32, hasher: crc32.NewIEEE()},
{name: "crc32c", hashType: HashTypeCRC32C, hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
{name: "crc64nvme", hashType: HashTypeCRC64NVME, hasher: crc64.New(crc64NVMETable)},
{name: "xxhash64", hashType: HashTypeXXHASH64, hasher: xxhash.New()},
{name: "xxhash3", hashType: HashTypeXXHASH3, hasher: xxh3.New()},
{name: "xxhash128", hashType: HashTypeXXHASH128, hasher: xxh3.New128()},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
expected := base64HashForTest(tt.hasher, data)
rdr, err := NewHashReader(bytes.NewReader(data), expected, tt.hashType)
if !assert.NoError(t, err) {
return
}
_, err = io.Copy(io.Discard, rdr)
assert.NoError(t, err)
assert.Equal(t, expected, rdr.Sum())
})
}
}
func TestNewCompositeChecksumReader_NewChecksumAlgorithms(t *testing.T) {
part1 := []byte("part one")
part2 := []byte("part two")
tests := []struct {
name string
hashType HashType
newHasher func() hash.Hash
}{
{name: "md5", hashType: HashTypeMd5, newHasher: md5.New},
{name: "sha1", hashType: HashTypeSha1, newHasher: sha1.New},
{name: "sha256", hashType: HashTypeSha256, newHasher: sha256.New},
{name: "sha512", hashType: HashTypeSha512, newHasher: sha512.New},
{name: "crc32", hashType: HashTypeCRC32, newHasher: func() hash.Hash { return crc32.NewIEEE() }},
{name: "crc32c", hashType: HashTypeCRC32C, newHasher: func() hash.Hash { return crc32.New(crc32.MakeTable(crc32.Castagnoli)) }},
{name: "xxhash64", hashType: HashTypeXXHASH64, newHasher: func() hash.Hash { return xxhash.New() }},
{name: "xxhash3", hashType: HashTypeXXHASH3, newHasher: func() hash.Hash { return xxh3.New() }},
{name: "xxhash128", hashType: HashTypeXXHASH128, newHasher: func() hash.Hash { return xxh3.New128() }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
part1Sum := base64HashForTest(tt.newHasher(), part1)
part2Sum := base64HashForTest(tt.newHasher(), part2)
composite, err := NewCompositeChecksumReader(tt.hashType)
if !assert.NoError(t, err) {
return
}
assert.NoError(t, composite.Process(part1Sum))
assert.NoError(t, composite.Process(part2Sum))
part1Bytes, err := base64.StdEncoding.DecodeString(part1Sum)
if !assert.NoError(t, err) {
return
}
part2Bytes, err := base64.StdEncoding.DecodeString(part2Sum)
if !assert.NoError(t, err) {
return
}
expectedHasher := tt.newHasher()
expectedHasher.Write(part1Bytes)
expectedHasher.Write(part2Bytes)
expected := base64.StdEncoding.EncodeToString(expectedHasher.Sum(nil))
assert.Equal(t, expected, composite.Sum())
})
}
}
s3api/utils/unsigned-chunk-reader.go
+14
View File
@@ -17,8 +17,10 @@ package utils
import (
"bufio"
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"errors"
"hash"
@@ -30,8 +32,10 @@ import (
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/cespare/xxhash/v2"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
"github.com/zeebo/xxh3"
)
var (
@@ -353,6 +357,16 @@ func getHasher(ct checksumType) (hash.Hash, error) {
return sha1.New(), nil
case checksumTypeSha256:
return sha256.New(), nil
case checksumTypeSha512:
return sha512.New(), nil
case checksumTypeMd5:
return md5.New(), nil
case checksumTypeXxhash64:
return xxhash.New(), nil
case checksumTypeXxhash3:
return xxh3.New(), nil
case checksumTypeXxhash128:
return xxh3.New128(), nil
default:
return nil, errors.New("unsupported checksum type")
}
s3api/utils/utils.go
+31 -1
View File
@@ -721,8 +721,13 @@ var checksumLengths = map[types.ChecksumAlgorithm]int{
types.ChecksumAlgorithmCrc32: 4,
types.ChecksumAlgorithmCrc32c: 4,
types.ChecksumAlgorithmCrc64nvme: 8,
types.ChecksumAlgorithmMd5: 16,
types.ChecksumAlgorithmSha1: 20,
types.ChecksumAlgorithmSha256: 32,
types.ChecksumAlgorithmSha512: 64,
types.ChecksumAlgorithmXxhash64: 8,
types.ChecksumAlgorithmXxhash3: 8,
types.ChecksumAlgorithmXxhash128: 16,
}
func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
@@ -753,7 +758,12 @@ func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
alg != types.ChecksumAlgorithmCrc32c &&
alg != types.ChecksumAlgorithmSha1 &&
alg != types.ChecksumAlgorithmSha256 &&
alg != types.ChecksumAlgorithmCrc64nvme {
alg != types.ChecksumAlgorithmCrc64nvme &&
alg != types.ChecksumAlgorithmSha512 &&
alg != types.ChecksumAlgorithmMd5 &&
alg != types.ChecksumAlgorithmXxhash64 &&
alg != types.ChecksumAlgorithmXxhash3 &&
alg != types.ChecksumAlgorithmXxhash128 {
debuglogger.Logf("invalid checksum algorithm: %v\n", alg)
return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
}
@@ -799,6 +809,26 @@ var checksumMap checksumSchema = checksumSchema{
types.ChecksumTypeFullObject: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmSha512: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmMd5: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash64: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash3: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
types.ChecksumAlgorithmXxhash128: checksumTypeSchema{
types.ChecksumTypeComposite: struct{}{},
"": struct{}{},
},
// Both could be empty
"": checksumTypeSchema{
"": struct{}{},
s3api/utils/utils_test.go
+211
View File
@@ -707,6 +707,41 @@ func TestIsChecksumAlgorithmValid(t *testing.T) {
},
wantErr: false,
},
{
name: "sha512",
args: args{
alg: types.ChecksumAlgorithmSha512,
},
wantErr: false,
},
{
name: "md5",
args: args{
alg: types.ChecksumAlgorithmMd5,
},
wantErr: false,
},
{
name: "xxhash64",
args: args{
alg: types.ChecksumAlgorithmXxhash64,
},
wantErr: false,
},
{
name: "xxhash3",
args: args{
alg: types.ChecksumAlgorithmXxhash3,
},
wantErr: false,
},
{
name: "xxhash128",
args: args{
alg: types.ChecksumAlgorithmXxhash128,
},
wantErr: false,
},
{
name: "invalid",
args: args{
@@ -806,6 +841,102 @@ func TestIsValidChecksum(t *testing.T) {
},
want: true,
},
{
name: "invalid-crc64nvme",
args: args{
checksum: "ww2FVQ==",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: false,
},
{
name: "valid-crc64nvme",
args: args{
checksum: "AAAAAAAAAAA=",
algorithm: types.ChecksumAlgorithmCrc64nvme,
},
want: true,
},
{
name: "invalid-sha512",
args: args{
checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
algorithm: types.ChecksumAlgorithmSha512,
},
want: false,
},
{
name: "valid-sha512",
args: args{
checksum: "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==",
algorithm: types.ChecksumAlgorithmSha512,
},
want: true,
},
{
name: "invalid-md5",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmMd5,
},
want: false,
},
{
name: "valid-md5",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmMd5,
},
want: true,
},
{
name: "invalid-xxhash64",
args: args{
checksum: "1B2M2Y8AsgTpgAmY7PhCfg==",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: false,
},
{
name: "valid-xxhash64",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash64,
},
want: true,
},
{
name: "invalid-xxhash3",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: false,
},
{
name: "valid-xxhash3",
args: args{
checksum: "LQaABTjTlMI=",
algorithm: types.ChecksumAlgorithmXxhash3,
},
want: true,
},
{
name: "invalid-xxhash128",
args: args{
checksum: "70bbN1HY6Zk=",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: false,
},
{
name: "valid-xxhash128",
args: args{
checksum: "maoG0wFHmNhgAcMkRo1Jfw==",
algorithm: types.ChecksumAlgorithmXxhash128,
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -1090,6 +1221,86 @@ func Test_checkChecksumTypeAndAlgo(t *testing.T) {
},
wantErr: true,
},
{
name: "full_object-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "full_object-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeFullObject,
},
wantErr: true,
},
{
name: "composite-sha512",
args: args{
algo: types.ChecksumAlgorithmSha512,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-md5",
args: args{
algo: types.ChecksumAlgorithmMd5,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash64",
args: args{
algo: types.ChecksumAlgorithmXxhash64,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash3",
args: args{
algo: types.ChecksumAlgorithmXxhash3,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-xxhash128",
args: args{
algo: types.ChecksumAlgorithmXxhash128,
t: types.ChecksumTypeComposite,
},
wantErr: false,
},
{
name: "composite-empty",
args: args{