mirror of
https://github.com/versity/versitygw.git
synced 2026-08-20 22:26:19 +00:00
feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support. These external packages have been used: - `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128` - `github.com/cespare/xxhash/v2` for `XXHASH64` Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types. References: - `https://github.com/aws/aws-sdk-go-v2/issues/3404` - `https://github.com/aws/aws-sdk-go-v2/issues/3403`
This commit is contained in:
+31
-1
@@ -721,8 +721,13 @@ var checksumLengths = map[types.ChecksumAlgorithm]int{
|
||||
types.ChecksumAlgorithmCrc32: 4,
|
||||
types.ChecksumAlgorithmCrc32c: 4,
|
||||
types.ChecksumAlgorithmCrc64nvme: 8,
|
||||
types.ChecksumAlgorithmMd5: 16,
|
||||
types.ChecksumAlgorithmSha1: 20,
|
||||
types.ChecksumAlgorithmSha256: 32,
|
||||
types.ChecksumAlgorithmSha512: 64,
|
||||
types.ChecksumAlgorithmXxhash64: 8,
|
||||
types.ChecksumAlgorithmXxhash3: 8,
|
||||
types.ChecksumAlgorithmXxhash128: 16,
|
||||
}
|
||||
|
||||
func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
|
||||
@@ -753,7 +758,12 @@ func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
|
||||
alg != types.ChecksumAlgorithmCrc32c &&
|
||||
alg != types.ChecksumAlgorithmSha1 &&
|
||||
alg != types.ChecksumAlgorithmSha256 &&
|
||||
alg != types.ChecksumAlgorithmCrc64nvme {
|
||||
alg != types.ChecksumAlgorithmCrc64nvme &&
|
||||
alg != types.ChecksumAlgorithmSha512 &&
|
||||
alg != types.ChecksumAlgorithmMd5 &&
|
||||
alg != types.ChecksumAlgorithmXxhash64 &&
|
||||
alg != types.ChecksumAlgorithmXxhash3 &&
|
||||
alg != types.ChecksumAlgorithmXxhash128 {
|
||||
debuglogger.Logf("invalid checksum algorithm: %v\n", alg)
|
||||
return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
|
||||
}
|
||||
@@ -799,6 +809,26 @@ var checksumMap checksumSchema = checksumSchema{
|
||||
types.ChecksumTypeFullObject: struct{}{},
|
||||
"": struct{}{},
|
||||
},
|
||||
types.ChecksumAlgorithmSha512: checksumTypeSchema{
|
||||
types.ChecksumTypeComposite: struct{}{},
|
||||
"": struct{}{},
|
||||
},
|
||||
types.ChecksumAlgorithmMd5: checksumTypeSchema{
|
||||
types.ChecksumTypeComposite: struct{}{},
|
||||
"": struct{}{},
|
||||
},
|
||||
types.ChecksumAlgorithmXxhash64: checksumTypeSchema{
|
||||
types.ChecksumTypeComposite: struct{}{},
|
||||
"": struct{}{},
|
||||
},
|
||||
types.ChecksumAlgorithmXxhash3: checksumTypeSchema{
|
||||
types.ChecksumTypeComposite: struct{}{},
|
||||
"": struct{}{},
|
||||
},
|
||||
types.ChecksumAlgorithmXxhash128: checksumTypeSchema{
|
||||
types.ChecksumTypeComposite: struct{}{},
|
||||
"": struct{}{},
|
||||
},
|
||||
// Both could be empty
|
||||
"": checksumTypeSchema{
|
||||
"": struct{}{},
|
||||
|
||||
Reference in New Issue
Block a user