From d2fa265fb876b84eede9f04fbf470751cae0ceb8 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Thu, 30 Apr 2026 22:24:07 +0400 Subject: [PATCH] feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support. These external packages have been used: - `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128` - `github.com/cespare/xxhash/v2` for `XXHASH64` Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types. References: - `https://github.com/aws/aws-sdk-go-v2/issues/3404` - `https://github.com/aws/aws-sdk-go-v2/issues/3403` --- backend/posix/posix.go | 279 +++++++--- backend/s3proxy/s3.go | 75 +++ go.mod | 2 + go.sum | 6 + s3api/controllers/bucket-post.go | 10 + s3api/controllers/bucket-post_test.go | 5 + s3api/controllers/object-get.go | 5 + s3api/controllers/object-get_test.go | 10 + s3api/controllers/object-head.go | 5 + s3api/controllers/object-head_test.go | 10 + s3api/controllers/object-post.go | 5 + s3api/controllers/object-put.go | 20 + s3api/controllers/object-put_test.go | 20 + s3api/middlewares/checksum.go | 4 +- s3api/utils/chunk-reader.go | 12 +- s3api/utils/csum-reader.go | 81 ++- s3api/utils/csum-reader_test.go | 99 ++++ s3api/utils/unsigned-chunk-reader.go | 14 + s3api/utils/utils.go | 32 +- s3api/utils/utils_test.go | 211 ++++++++ s3err/s3err.go | 2 +- s3response/s3response.go | 35 ++ tests/integration/CompleteMultipartUpload.go | 216 ++++---- tests/integration/CopyObject.go | 36 +- tests/integration/CreateMultipartUpload.go | 10 + tests/integration/GetObject.go | 176 ++----- tests/integration/GetObjectAttributes.go | 64 +-- tests/integration/HeadObject.go | 90 +--- tests/integration/ListMultipartUploads.go | 26 + tests/integration/ListObjects.go | 2 +- tests/integration/ListObjectsV2.go | 2 +- tests/integration/PostObject.go | 5 + tests/integration/PutObject.go | 243 +++------ tests/integration/UploadPart.go | 264 ++-------- tests/integration/UploadPartCopy.go | 193 +++---- tests/integration/group-tests.go | 2 - .../signed_streaming_payload_trailer.go | 50 +- .../unsigned_streaming_payload_trailer.go | 39 +- tests/integration/utils.go | 476 +++++++++++++++++- 39 files changed, 1811 insertions(+), 1025 deletions(-) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 680dfa38..9d388ace 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1659,11 +1659,79 @@ func getPartChecksum(algo types.ChecksumAlgorithm, part types.CompletedPart) str return backend.GetStringFromPtr(part.ChecksumSHA256) case types.ChecksumAlgorithmCrc64nvme: return backend.GetStringFromPtr(part.ChecksumCRC64NVME) + case types.ChecksumAlgorithmSha512: + return backend.GetStringFromPtr(part.ChecksumSHA512) + case types.ChecksumAlgorithmMd5: + return backend.GetStringFromPtr(part.ChecksumMD5) + case types.ChecksumAlgorithmXxhash64: + return backend.GetStringFromPtr(part.ChecksumXXHASH64) + case types.ChecksumAlgorithmXxhash3: + return backend.GetStringFromPtr(part.ChecksumXXHASH3) + case types.ChecksumAlgorithmXxhash128: + return backend.GetStringFromPtr(part.ChecksumXXHASH128) default: return "" } } +func setStoredChecksum(checksum *s3response.Checksum, algo types.ChecksumAlgorithm, sum *string) { + if sum == nil { + return + } + + switch algo { + case types.ChecksumAlgorithmCrc32: + checksum.CRC32 = sum + case types.ChecksumAlgorithmCrc32c: + checksum.CRC32C = sum + case types.ChecksumAlgorithmSha1: + checksum.SHA1 = sum + case types.ChecksumAlgorithmSha256: + checksum.SHA256 = sum + case types.ChecksumAlgorithmCrc64nvme: + checksum.CRC64NVME = sum + case types.ChecksumAlgorithmSha512: + checksum.SHA512 = sum + case types.ChecksumAlgorithmMd5: + checksum.MD5 = sum + case types.ChecksumAlgorithmXxhash64: + checksum.XXHASH64 = sum + case types.ChecksumAlgorithmXxhash3: + checksum.XXHASH3 = sum + case types.ChecksumAlgorithmXxhash128: + checksum.XXHASH128 = sum + } +} + +func setUploadPartChecksum(res *s3.UploadPartOutput, algo types.ChecksumAlgorithm, sum *string) { + if sum == nil { + return + } + + switch algo { + case types.ChecksumAlgorithmCrc32: + res.ChecksumCRC32 = sum + case types.ChecksumAlgorithmCrc32c: + res.ChecksumCRC32C = sum + case types.ChecksumAlgorithmSha1: + res.ChecksumSHA1 = sum + case types.ChecksumAlgorithmSha256: + res.ChecksumSHA256 = sum + case types.ChecksumAlgorithmCrc64nvme: + res.ChecksumCRC64NVME = sum + case types.ChecksumAlgorithmSha512: + res.ChecksumSHA512 = sum + case types.ChecksumAlgorithmMd5: + res.ChecksumMD5 = sum + case types.ChecksumAlgorithmXxhash64: + res.ChecksumXXHASH64 = sum + case types.ChecksumAlgorithmXxhash3: + res.ChecksumXXHASH3 = sum + case types.ChecksumAlgorithmXxhash128: + res.ChecksumXXHASH128 = sum + } +} + func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) { release, err := p.acquireActionSlot(ctx) if err != nil { @@ -1947,6 +2015,11 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C var sha1 *string var sha256 *string var crc64nvme *string + var sha512 *string + var md5sum *string + var xxhash64 *string + var xxhash3 *string + var xxhash128 *string var gotSum *string switch checksums.Algorithm { @@ -1970,6 +2043,26 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C gotSum = input.ChecksumCRC64NVME checksums.CRC64NVME = &value crc64nvme = &value + case types.ChecksumAlgorithmSha512: + gotSum = input.ChecksumSHA512 + checksums.SHA512 = &value + sha512 = &value + case types.ChecksumAlgorithmMd5: + gotSum = input.ChecksumMD5 + checksums.MD5 = &value + md5sum = &value + case types.ChecksumAlgorithmXxhash64: + gotSum = input.ChecksumXXHASH64 + checksums.XXHASH64 = &value + xxhash64 = &value + case types.ChecksumAlgorithmXxhash3: + gotSum = input.ChecksumXXHASH3 + checksums.XXHASH3 = &value + xxhash3 = &value + case types.ChecksumAlgorithmXxhash128: + gotSum = input.ChecksumXXHASH128 + checksums.XXHASH128 = &value + xxhash128 = &value } // Check if the provided checksum and the calculated one are the same. @@ -2176,6 +2269,11 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C ChecksumSHA1: sha1, ChecksumSHA256: sha256, ChecksumCRC64NVME: crc64nvme, + ChecksumSHA512: sha512, + ChecksumMD5: md5sum, + ChecksumXXHASH64: xxhash64, + ChecksumXXHASH3: xxhash3, + ChecksumXXHASH128: xxhash128, ChecksumType: &checksums.Type, }, versionID, nil } @@ -2212,6 +2310,11 @@ func validatePartChecksum(checksum s3response.Checksum, part types.CompletedPart {part.ChecksumSHA1, getString(checksum.SHA1), types.ChecksumAlgorithmSha1}, {part.ChecksumSHA256, getString(checksum.SHA256), types.ChecksumAlgorithmSha256}, {part.ChecksumCRC64NVME, getString(checksum.CRC64NVME), types.ChecksumAlgorithmCrc64nvme}, + {part.ChecksumSHA512, getString(checksum.SHA512), types.ChecksumAlgorithmSha512}, + {part.ChecksumMD5, getString(checksum.MD5), types.ChecksumAlgorithmMd5}, + {part.ChecksumXXHASH64, getString(checksum.XXHASH64), types.ChecksumAlgorithmXxhash64}, + {part.ChecksumXXHASH3, getString(checksum.XXHASH3), types.ChecksumAlgorithmXxhash3}, + {part.ChecksumXXHASH128, getString(checksum.XXHASH128), types.ChecksumAlgorithmXxhash128}, } { if cs.checksum == nil { continue @@ -2254,6 +2357,21 @@ func numberOfChecksums(part types.CompletedPart) int { if getString(part.ChecksumCRC64NVME) != "" { counter++ } + if getString(part.ChecksumSHA512) != "" { + counter++ + } + if getString(part.ChecksumMD5) != "" { + counter++ + } + if getString(part.ChecksumXXHASH64) != "" { + counter++ + } + if getString(part.ChecksumXXHASH3) != "" { + counter++ + } + if getString(part.ChecksumXXHASH128) != "" { + counter++ + } return counter } @@ -2778,6 +2896,11 @@ func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3resp ChecksumSHA1: checksum.SHA1, ChecksumSHA256: checksum.SHA256, ChecksumCRC64NVME: checksum.CRC64NVME, + ChecksumSHA512: checksum.SHA512, + ChecksumMD5: checksum.MD5, + ChecksumXXHASH64: checksum.XXHASH64, + ChecksumXXHASH3: checksum.XXHASH3, + ChecksumXXHASH128: checksum.XXHASH128, }) } @@ -2899,6 +3022,11 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart {input.ChecksumSHA1, utils.HashTypeSha1}, {input.ChecksumSHA256, utils.HashTypeSha256}, {input.ChecksumCRC64NVME, utils.HashTypeCRC64NVME}, + {input.ChecksumSHA512, utils.HashTypeSha512}, + {input.ChecksumMD5, utils.HashTypeMd5}, + {input.ChecksumXXHASH64, utils.HashTypeXXHASH64}, + {input.ChecksumXXHASH3, utils.HashTypeXXHASH3}, + {input.ChecksumXXHASH128, utils.HashTypeXXHASH128}, } for _, config := range hashConfigs { @@ -3024,24 +3152,8 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart sum = hashRdr.Sum() } - // Assign the checksum - switch checksums.Algorithm { - case types.ChecksumAlgorithmCrc32: - checksum.CRC32 = &sum - res.ChecksumCRC32 = &sum - case types.ChecksumAlgorithmCrc32c: - checksum.CRC32C = &sum - res.ChecksumCRC32C = &sum - case types.ChecksumAlgorithmSha1: - checksum.SHA1 = &sum - res.ChecksumSHA1 = &sum - case types.ChecksumAlgorithmSha256: - checksum.SHA256 = &sum - res.ChecksumSHA256 = &sum - case types.ChecksumAlgorithmCrc64nvme: - checksum.CRC64NVME = &sum - res.ChecksumCRC64NVME = &sum - } + setStoredChecksum(&checksum, checksums.Algorithm, &sum) + setUploadPartChecksum(res, checksums.Algorithm, &sum) err := p.storeChecksums(f.File(), bucket, partPath, checksum) if err != nil { @@ -3083,6 +3195,16 @@ func (p *Posix) UploadPartWithPostFunc(ctx context.Context, input *s3.UploadPart res.ChecksumSHA256 = &sumToReturn case utils.HashTypeCRC64NVME: res.ChecksumCRC64NVME = &sumToReturn + case utils.HashTypeSha512: + res.ChecksumSHA512 = &sumToReturn + case utils.HashTypeMd5: + res.ChecksumMD5 = &sumToReturn + case utils.HashTypeXXHASH64: + res.ChecksumXXHASH64 = &sumToReturn + case utils.HashTypeXXHASH3: + res.ChecksumXXHASH3 = &sumToReturn + case utils.HashTypeXXHASH128: + res.ChecksumXXHASH128 = &sumToReturn } } } @@ -3321,18 +3443,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) } sum := hashRdr.Sum() - switch algo { - case types.ChecksumAlgorithmCrc32: - checksums.CRC32 = &sum - case types.ChecksumAlgorithmCrc32c: - checksums.CRC32C = &sum - case types.ChecksumAlgorithmSha1: - checksums.SHA1 = &sum - case types.ChecksumAlgorithmSha256: - checksums.SHA256 = &sum - case types.ChecksumAlgorithmCrc64nvme: - checksums.CRC64NVME = &sum - } + setStoredChecksum(&checksums, algo, &sum) err := p.storeChecksums(f.File(), *upi.Bucket, partPath, checksums) if err != nil { @@ -3374,6 +3485,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) ChecksumSHA1: checksums.SHA1, ChecksumSHA256: checksums.SHA256, ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, }, nil } @@ -3389,6 +3505,16 @@ func getEmptyChecksumValue(algo types.ChecksumAlgorithm) string { return "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" case types.ChecksumAlgorithmSha256: return "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + case types.ChecksumAlgorithmSha512: + return "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" + case types.ChecksumAlgorithmMd5: + return "1B2M2Y8AsgTpgAmY7PhCfg==" + case types.ChecksumAlgorithmXxhash64: + return "70bbN1HY6Zk=" + case types.ChecksumAlgorithmXxhash3: + return "LQaABTjTlMI=" + case types.ChecksumAlgorithmXxhash128: + return "maoG0wFHmNhgAcMkRo1Jfw==" default: // default to crc64nvme return "AAAAAAAAAAA=" @@ -3464,6 +3590,11 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje {po.ChecksumSHA1, utils.HashTypeSha1}, {po.ChecksumSHA256, utils.HashTypeSha256}, {po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME}, + {po.ChecksumSHA512, utils.HashTypeSha512}, + {po.ChecksumMD5, utils.HashTypeMd5}, + {po.ChecksumXXHASH64, utils.HashTypeXXHASH64}, + {po.ChecksumXXHASH3, utils.HashTypeXXHASH3}, + {po.ChecksumXXHASH128, utils.HashTypeXXHASH128}, } for _, config := range hashConfigs { @@ -3535,19 +3666,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje Algorithm: checksumAlgorithm, } - // Store the calculated checksum in the object metadata - switch checksumAlgorithm { - case types.ChecksumAlgorithmCrc32: - checksum.CRC32 = &expectedSum - case types.ChecksumAlgorithmCrc32c: - checksum.CRC32C = &expectedSum - case types.ChecksumAlgorithmSha1: - checksum.SHA1 = &expectedSum - case types.ChecksumAlgorithmSha256: - checksum.SHA256 = &expectedSum - case types.ChecksumAlgorithmCrc64nvme: - checksum.CRC64NVME = &expectedSum - } + setStoredChecksum(&checksum, checksumAlgorithm, &expectedSum) err = p.storeChecksums(nil, *po.Bucket, *po.Key, checksum) if err != nil { @@ -3564,6 +3683,11 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje ChecksumCRC64NVME: checksum.CRC64NVME, ChecksumSHA1: checksum.SHA1, ChecksumSHA256: checksum.SHA256, + ChecksumSHA512: checksum.SHA512, + ChecksumMD5: checksum.MD5, + ChecksumXXHASH64: checksum.XXHASH64, + ChecksumXXHASH3: checksum.XXHASH3, + ChecksumXXHASH128: checksum.XXHASH128, }, nil } @@ -3709,19 +3833,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje Algorithm: checksumAlgorithm, } - // Store the calculated checksum in the object metadata - switch checksumAlgorithm { - case types.ChecksumAlgorithmCrc32: - checksum.CRC32 = &sum - case types.ChecksumAlgorithmCrc32c: - checksum.CRC32C = &sum - case types.ChecksumAlgorithmSha1: - checksum.SHA1 = &sum - case types.ChecksumAlgorithmSha256: - checksum.SHA256 = &sum - case types.ChecksumAlgorithmCrc64nvme: - checksum.CRC64NVME = &sum - } + setStoredChecksum(&checksum, checksumAlgorithm, &sum) err = p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) @@ -3822,6 +3934,11 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje ChecksumSHA1: checksum.SHA1, ChecksumSHA256: checksum.SHA256, ChecksumCRC64NVME: checksum.CRC64NVME, + ChecksumSHA512: checksum.SHA512, + ChecksumMD5: checksum.MD5, + ChecksumXXHASH64: checksum.XXHASH64, + ChecksumXXHASH3: checksum.XXHASH3, + ChecksumXXHASH128: checksum.XXHASH128, Size: &objsize, ChecksumType: checksum.Type, }, nil @@ -4423,6 +4540,11 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge ChecksumSHA1: checksums.SHA1, ChecksumSHA256: checksums.SHA256, ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, ChecksumType: checksums.Type, AcceptRanges: backend.GetPtrFromString("bytes"), ContentLength: &length, @@ -4586,6 +4708,11 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge ChecksumSHA1: checksums.SHA1, ChecksumSHA256: checksums.SHA256, ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, ChecksumType: checksums.Type, PartsCount: partsCount, }, nil @@ -4837,6 +4964,11 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ChecksumSHA1: checksums.SHA1, ChecksumSHA256: checksums.SHA256, ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumSHA512: checksums.SHA512, + ChecksumMD5: checksums.MD5, + ChecksumXXHASH64: checksums.XXHASH64, + ChecksumXXHASH3: checksums.XXHASH3, + ChecksumXXHASH128: checksums.XXHASH128, ChecksumType: checksums.Type, TagCount: tagCount, PartsCount: partsCount, @@ -4880,6 +5012,11 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr ChecksumSHA1: data.ChecksumSHA1, ChecksumSHA256: data.ChecksumSHA256, ChecksumCRC64NVME: data.ChecksumCRC64NVME, + ChecksumSHA512: data.ChecksumSHA512, + ChecksumMD5: data.ChecksumMD5, + ChecksumXXHASH64: data.ChecksumXXHASH64, + ChecksumXXHASH3: data.ChecksumXXHASH3, + ChecksumXXHASH128: data.ChecksumXXHASH128, ChecksumType: data.ChecksumType, }, }, nil @@ -5026,6 +5163,11 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput var sha1 *string var sha256 *string var crc64nvme *string + var sha512 *string + var md5sum *string + var xxhash64 *string + var xxhash3 *string + var xxhash128 *string var chType types.ChecksumType dstObjdPath := joinPathWithTrailer(dstBucket, dstObject) @@ -5086,6 +5228,21 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput case utils.HashTypeCRC64NVME: checksums.CRC64NVME = &sum crc64nvme = &sum + case utils.HashTypeSha512: + checksums.SHA512 = &sum + sha512 = &sum + case utils.HashTypeMd5: + checksums.MD5 = &sum + md5sum = &sum + case utils.HashTypeXXHASH64: + checksums.XXHASH64 = &sum + xxhash64 = &sum + case utils.HashTypeXXHASH3: + checksums.XXHASH3 = &sum + xxhash3 = &sum + case utils.HashTypeXXHASH128: + checksums.XXHASH128 = &sum + xxhash128 = &sum } // If a new checksum is calculated, the checksum type @@ -5209,6 +5366,11 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput sha1 = res.ChecksumSHA1 sha256 = res.ChecksumSHA256 crc64nvme = res.ChecksumCRC64NVME + sha512 = res.ChecksumSHA512 + md5sum = res.ChecksumMD5 + xxhash64 = res.ChecksumXXHASH64 + xxhash3 = res.ChecksumXXHASH3 + xxhash128 = res.ChecksumXXHASH128 chType = res.ChecksumType } @@ -5226,6 +5388,11 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput ChecksumSHA1: sha1, ChecksumSHA256: sha256, ChecksumCRC64NVME: crc64nvme, + ChecksumSHA512: sha512, + ChecksumMD5: md5sum, + ChecksumXXHASH64: xxhash64, + ChecksumXXHASH3: xxhash3, + ChecksumXXHASH128: xxhash128, ChecksumType: chType, }, VersionId: version, diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index 6e603179..fb703d7f 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -483,6 +483,21 @@ func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.Complet if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" { input.ChecksumSHA256 = nil } + if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" { + input.ChecksumSHA512 = nil + } + if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" { + input.ChecksumMD5 = nil + } + if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" { + input.ChecksumXXHASH64 = nil + } + if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" { + input.ChecksumXXHASH3 = nil + } + if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" { + input.ChecksumXXHASH128 = nil + } if input.ExpectedBucketOwner != nil && *input.ExpectedBucketOwner == "" { input.ExpectedBucketOwner = nil } @@ -518,6 +533,11 @@ func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.Complet ChecksumCRC64NVME: out.ChecksumCRC64NVME, ChecksumSHA1: out.ChecksumSHA1, ChecksumSHA256: out.ChecksumSHA256, + ChecksumSHA512: out.ChecksumSHA512, + ChecksumMD5: out.ChecksumMD5, + ChecksumXXHASH64: out.ChecksumXXHASH64, + ChecksumXXHASH3: out.ChecksumXXHASH3, + ChecksumXXHASH128: out.ChecksumXXHASH128, ChecksumType: &out.ChecksumType, } if out.VersionId != nil { @@ -653,6 +673,11 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re ChecksumCRC64NVME: p.ChecksumCRC64NVME, ChecksumSHA1: p.ChecksumSHA1, ChecksumSHA256: p.ChecksumSHA256, + ChecksumSHA512: p.ChecksumSHA512, + ChecksumMD5: p.ChecksumMD5, + ChecksumXXHASH64: p.ChecksumXXHASH64, + ChecksumXXHASH3: p.ChecksumXXHASH3, + ChecksumXXHASH128: p.ChecksumXXHASH128, }) } pnm, err := strconv.Atoi(*output.PartNumberMarker) @@ -709,6 +734,21 @@ func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" { input.ChecksumSHA256 = nil } + if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" { + input.ChecksumSHA512 = nil + } + if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" { + input.ChecksumMD5 = nil + } + if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" { + input.ChecksumXXHASH64 = nil + } + if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" { + input.ChecksumXXHASH3 = nil + } + if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" { + input.ChecksumXXHASH128 = nil + } if input.ContentMD5 != nil && *input.ContentMD5 == "" { input.ContentMD5 = nil } @@ -790,6 +830,11 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME, ChecksumSHA1: output.CopyPartResult.ChecksumSHA1, ChecksumSHA256: output.CopyPartResult.ChecksumSHA256, + ChecksumSHA512: output.CopyPartResult.ChecksumSHA512, + ChecksumMD5: output.CopyPartResult.ChecksumMD5, + ChecksumXXHASH64: output.CopyPartResult.ChecksumXXHASH64, + ChecksumXXHASH3: output.CopyPartResult.ChecksumXXHASH3, + ChecksumXXHASH128: output.CopyPartResult.ChecksumXXHASH128, }, nil } @@ -815,6 +860,21 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 == "" { input.ChecksumSHA256 = nil } + if input.ChecksumSHA512 != nil && *input.ChecksumSHA512 == "" { + input.ChecksumSHA512 = nil + } + if input.ChecksumMD5 != nil && *input.ChecksumMD5 == "" { + input.ChecksumMD5 = nil + } + if input.ChecksumXXHASH64 != nil && *input.ChecksumXXHASH64 == "" { + input.ChecksumXXHASH64 = nil + } + if input.ChecksumXXHASH3 != nil && *input.ChecksumXXHASH3 == "" { + input.ChecksumXXHASH3 = nil + } + if input.ChecksumXXHASH128 != nil && *input.ChecksumXXHASH128 == "" { + input.ChecksumXXHASH128 = nil + } if input.ContentDisposition != nil && *input.ContentDisposition == "" { input.ContentDisposition = nil } @@ -913,6 +973,11 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput ChecksumSHA1: input.ChecksumSHA1, ChecksumSHA256: input.ChecksumSHA256, ChecksumCRC64NVME: input.ChecksumCRC64NVME, + ChecksumSHA512: input.ChecksumSHA512, + ChecksumMD5: input.ChecksumMD5, + ChecksumXXHASH64: input.ChecksumXXHASH64, + ChecksumXXHASH3: input.ChecksumXXHASH3, + ChecksumXXHASH128: input.ChecksumXXHASH128, ContentMD5: input.ContentMD5, ExpectedBucketOwner: input.ExpectedBucketOwner, GrantFullControl: input.GrantFullControl, @@ -947,6 +1012,11 @@ func (s *S3Proxy) PutObject(ctx context.Context, input s3response.PutObjectInput ChecksumCRC64NVME: output.ChecksumCRC64NVME, ChecksumSHA1: output.ChecksumSHA1, ChecksumSHA256: output.ChecksumSHA256, + ChecksumSHA512: output.ChecksumSHA512, + ChecksumMD5: output.ChecksumMD5, + ChecksumXXHASH64: output.ChecksumXXHASH64, + ChecksumXXHASH3: output.ChecksumXXHASH3, + ChecksumXXHASH128: output.ChecksumXXHASH128, Size: output.Size, }, nil } @@ -1292,6 +1362,11 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input s3response.CopyObjectInp ChecksumCRC64NVME: out.CopyObjectResult.ChecksumCRC64NVME, ChecksumSHA1: out.CopyObjectResult.ChecksumSHA1, ChecksumSHA256: out.CopyObjectResult.ChecksumSHA256, + ChecksumSHA512: out.CopyObjectResult.ChecksumSHA512, + ChecksumMD5: out.CopyObjectResult.ChecksumMD5, + ChecksumXXHASH64: out.CopyObjectResult.ChecksumXXHASH64, + ChecksumXXHASH3: out.CopyObjectResult.ChecksumXXHASH3, + ChecksumXXHASH128: out.CopyObjectResult.ChecksumXXHASH128, ChecksumType: out.CopyObjectResult.ChecksumType, ETag: out.CopyObjectResult.ETag, LastModified: out.CopyObjectResult.LastModified, diff --git a/go.mod b/go.mod index 631031f9..3cd49522 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.18 github.com/aws/aws-sdk-go-v2/service/s3 v1.100.0 github.com/aws/smithy-go v1.25.1 + github.com/cespare/xxhash/v2 v2.3.0 github.com/davecgh/go-spew v1.1.1 github.com/go-ldap/ldap/v3 v3.4.13 github.com/gofiber/fiber/v2 v2.52.13 @@ -30,6 +31,7 @@ require ( github.com/urfave/cli/v2 v2.27.7 github.com/valyala/fasthttp v1.70.0 github.com/versity/scoutfs-go v0.0.0-20240625221833-95fd765b760b + github.com/zeebo/xxh3 v1.1.0 golang.org/x/sync v0.20.0 golang.org/x/sys v0.43.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 5e76d283..fa87eb8d 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.0 h1:ks8KBcZPh3PYISr5dAiXCM5/Thcu github.com/aws/aws-sdk-go-v2/service/sts v1.42.0/go.mod h1:pFw33T0WLvXU3rw1WBkpMlkgIn54eCB5FYLhjDc9Foo= github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= @@ -194,6 +196,10 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/s3api/controllers/bucket-post.go b/s3api/controllers/bucket-post.go index 4857fddd..e22c420c 100644 --- a/s3api/controllers/bucket-post.go +++ b/s3api/controllers/bucket-post.go @@ -222,6 +222,11 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) { ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]), + ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]), + ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]), + ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]), + ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]), }) if err != nil { return &Response{ @@ -285,6 +290,11 @@ func (c S3ApiController) POSTObject(ctx *fiber.Ctx) (*Response, error) { "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, "x-amz-checksum-sha1": res.ChecksumSHA1, "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-sha512": res.ChecksumSHA512, + "x-amz-checksum-md5": res.ChecksumMD5, + "x-amz-checksum-xxhash64": res.ChecksumXXHASH64, + "x-amz-checksum-xxhash3": res.ChecksumXXHASH3, + "x-amz-checksum-xxhash128": res.ChecksumXXHASH128, "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), "x-amz-version-id": utils.GetStringPtr(res.VersionID), }, diff --git a/s3api/controllers/bucket-post_test.go b/s3api/controllers/bucket-post_test.go index 31e8d75e..dc76c1dc 100644 --- a/s3api/controllers/bucket-post_test.go +++ b/s3api/controllers/bucket-post_test.go @@ -547,6 +547,11 @@ func TestS3ApiController_POSTObject(t *testing.T) { "x-amz-checksum-crc64nvme": utils.GetStringPtr(""), "x-amz-checksum-sha1": utils.GetStringPtr(""), "x-amz-checksum-sha256": utils.GetStringPtr(""), + "x-amz-checksum-sha512": utils.GetStringPtr(""), + "x-amz-checksum-md5": utils.GetStringPtr(""), + "x-amz-checksum-xxhash64": utils.GetStringPtr(""), + "x-amz-checksum-xxhash3": utils.GetStringPtr(""), + "x-amz-checksum-xxhash128": utils.GetStringPtr(""), "x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)), "x-amz-version-id": utils.GetStringPtr("vid-123"), }, diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go index ca31bb91..62c616d6 100644 --- a/s3api/controllers/object-get.go +++ b/s3api/controllers/object-get.go @@ -556,6 +556,11 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { "x-amz-checksum-crc32c": res.ChecksumCRC32C, "x-amz-checksum-sha1": res.ChecksumSHA1, "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-sha512": res.ChecksumSHA512, + "x-amz-checksum-md5": res.ChecksumMD5, + "x-amz-checksum-xxhash64": res.ChecksumXXHASH64, + "x-amz-checksum-xxhash3": res.ChecksumXXHASH3, + "x-amz-checksum-xxhash128": res.ChecksumXXHASH128, "Content-Type": utils.ApplyOverride(res.ContentType, responseOverrides["Content-Type"]), "x-amz-version-id": res.VersionId, "Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength), diff --git a/s3api/controllers/object-get_test.go b/s3api/controllers/object-get_test.go index 6eedcad6..5349cdee 100644 --- a/s3api/controllers/object-get_test.go +++ b/s3api/controllers/object-get_test.go @@ -799,6 +799,11 @@ func TestS3ApiController_GetObject(t *testing.T) { "x-amz-checksum-crc32c": nil, "x-amz-checksum-sha1": nil, "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, "x-amz-version-id": nil, "x-amz-mp-parts-count": nil, "x-amz-object-lock-mode": nil, @@ -844,6 +849,11 @@ func TestS3ApiController_GetObject(t *testing.T) { "x-amz-checksum-crc32c": nil, "x-amz-checksum-sha1": nil, "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, "x-amz-version-id": nil, "x-amz-mp-parts-count": nil, "x-amz-object-lock-mode": nil, diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go index ef89cdbc..12c8078c 100644 --- a/s3api/controllers/object-head.go +++ b/s3api/controllers/object-head.go @@ -188,6 +188,11 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { "x-amz-checksum-crc32c": res.ChecksumCRC32C, "x-amz-checksum-sha1": res.ChecksumSHA1, "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-sha512": res.ChecksumSHA512, + "x-amz-checksum-md5": res.ChecksumMD5, + "x-amz-checksum-xxhash64": res.ChecksumXXHASH64, + "x-amz-checksum-xxhash3": res.ChecksumXXHASH3, + "x-amz-checksum-xxhash128": res.ChecksumXXHASH128, "x-amz-version-id": res.VersionId, "x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount), "x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode), diff --git a/s3api/controllers/object-head_test.go b/s3api/controllers/object-head_test.go index 7cc5d097..b3293ad6 100644 --- a/s3api/controllers/object-head_test.go +++ b/s3api/controllers/object-head_test.go @@ -189,6 +189,11 @@ func TestS3ApiController_HeadObject(t *testing.T) { "x-amz-checksum-crc32c": nil, "x-amz-checksum-sha1": nil, "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, "x-amz-version-id": nil, "x-amz-mp-parts-count": nil, "x-amz-object-lock-mode": nil, @@ -233,6 +238,11 @@ func TestS3ApiController_HeadObject(t *testing.T) { "x-amz-checksum-crc32c": nil, "x-amz-checksum-sha1": nil, "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, "x-amz-version-id": nil, "x-amz-mp-parts-count": nil, "x-amz-object-lock-mode": nil, diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go index 098ff187..9e767b7b 100644 --- a/s3api/controllers/object-post.go +++ b/s3api/controllers/object-post.go @@ -375,6 +375,11 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]), + ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]), + ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]), + ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]), + ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]), ChecksumType: checksumType, IfMatch: ifMatch, IfNoneMatch: ifNoneMatch, diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index 1228559f..cefef2c2 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -306,6 +306,11 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]), + ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]), + ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]), + ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]), + ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]), }) var headers map[string]*string if err == nil { @@ -316,6 +321,11 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, "x-amz-checksum-sha1": res.ChecksumSHA1, "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-sha512": res.ChecksumSHA512, + "x-amz-checksum-md5": res.ChecksumMD5, + "x-amz-checksum-xxhash64": res.ChecksumXXHASH64, + "x-amz-checksum-xxhash3": res.ChecksumXXHASH3, + "x-amz-checksum-xxhash128": res.ChecksumXXHASH128, } } return &Response{ @@ -789,6 +799,11 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + ChecksumSHA512: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha512]), + ChecksumMD5: utils.GetStringPtr(checksums[types.ChecksumAlgorithmMd5]), + ChecksumXXHASH64: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash64]), + ChecksumXXHASH3: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash3]), + ChecksumXXHASH128: utils.GetStringPtr(checksums[types.ChecksumAlgorithmXxhash128]), IfMatch: ifMatch, IfNoneMatch: ifNoneMatch, }) @@ -800,6 +815,11 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, "x-amz-checksum-sha1": res.ChecksumSHA1, "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-sha512": res.ChecksumSHA512, + "x-amz-checksum-md5": res.ChecksumMD5, + "x-amz-checksum-xxhash64": res.ChecksumXXHASH64, + "x-amz-checksum-xxhash3": res.ChecksumXXHASH3, + "x-amz-checksum-xxhash128": res.ChecksumXXHASH128, "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), "x-amz-version-id": &res.VersionID, "x-amz-object-size": utils.ConvertPtrToStringPtr(res.Size), diff --git a/s3api/controllers/object-put_test.go b/s3api/controllers/object-put_test.go index 09648fdb..c88274d5 100644 --- a/s3api/controllers/object-put_test.go +++ b/s3api/controllers/object-put_test.go @@ -535,6 +535,11 @@ func TestS3ApiController_UploadPart(t *testing.T) { "x-amz-checksum-crc64nvme": nil, "x-amz-checksum-sha1": nil, "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, }, MetaOpts: &MetaOptions{ BucketOwner: "root", @@ -1214,6 +1219,11 @@ func TestS3ApiController_PutObject(t *testing.T) { "x-amz-checksum-crc64nvme": nil, "x-amz-checksum-sha1": nil, "x-amz-checksum-sha256": nil, + "x-amz-checksum-sha512": nil, + "x-amz-checksum-md5": nil, + "x-amz-checksum-xxhash64": nil, + "x-amz-checksum-xxhash3": nil, + "x-amz-checksum-xxhash128": nil, "x-amz-checksum-type": nil, "x-amz-version-id": emptyStringPtr, "x-amz-object-size": nil, @@ -1255,6 +1265,11 @@ func TestS3ApiController_PutObject(t *testing.T) { ChecksumSHA1: utils.GetStringPtr("sha1"), ChecksumSHA256: utils.GetStringPtr("sha256"), ChecksumCRC64NVME: utils.GetStringPtr("crc64nvme"), + ChecksumSHA512: utils.GetStringPtr("sha512"), + ChecksumMD5: utils.GetStringPtr("md5"), + ChecksumXXHASH64: utils.GetStringPtr("xxhash64"), + ChecksumXXHASH3: utils.GetStringPtr("xxhash3"), + ChecksumXXHASH128: utils.GetStringPtr("xxhash128"), ChecksumType: types.ChecksumTypeComposite, VersionID: "versionId", Size: &objSize, @@ -1269,6 +1284,11 @@ func TestS3ApiController_PutObject(t *testing.T) { "x-amz-checksum-crc64nvme": utils.GetStringPtr("crc64nvme"), "x-amz-checksum-sha1": utils.GetStringPtr("sha1"), "x-amz-checksum-sha256": utils.GetStringPtr("sha256"), + "x-amz-checksum-sha512": utils.GetStringPtr("sha512"), + "x-amz-checksum-md5": utils.GetStringPtr("md5"), + "x-amz-checksum-xxhash64": utils.GetStringPtr("xxhash64"), + "x-amz-checksum-xxhash3": utils.GetStringPtr("xxhash3"), + "x-amz-checksum-xxhash128": utils.GetStringPtr("xxhash128"), "x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)), "x-amz-version-id": utils.GetStringPtr("versionId"), "x-amz-object-size": utils.ConvertToStringPtr(objSize), diff --git a/s3api/middlewares/checksum.go b/s3api/middlewares/checksum.go index 9ca12532..2b182487 100644 --- a/s3api/middlewares/checksum.go +++ b/s3api/middlewares/checksum.go @@ -49,7 +49,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi var err error wrapBodyReader(ctx, func(r io.Reader) io.Reader { - r, err = utils.NewHashReader(r, md5sum, utils.HashTypeMd5) + r, err = utils.NewHashReader(r, md5sum, utils.HashTypeContentMD5) return r }) if err != nil { @@ -70,7 +70,7 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi return s3err.GetAPIError(s3err.ErrInvalidDigest) } - rdr, err = utils.NewHashReader(bytes.NewReader(body), md5sum, utils.HashTypeMd5) + rdr, err = utils.NewHashReader(bytes.NewReader(body), md5sum, utils.HashTypeContentMD5) if err != nil { return err } diff --git a/s3api/utils/chunk-reader.go b/s3api/utils/chunk-reader.go index 05ee98c2..14f61da2 100644 --- a/s3api/utils/chunk-reader.go +++ b/s3api/utils/chunk-reader.go @@ -79,6 +79,11 @@ const ( checksumTypeSha1 checksumType = "x-amz-checksum-sha1" checksumTypeSha256 checksumType = "x-amz-checksum-sha256" checksumTypeCrc64nvme checksumType = "x-amz-checksum-crc64nvme" + checksumTypeSha512 checksumType = "x-amz-checksum-sha512" + checksumTypeMd5 checksumType = "x-amz-checksum-md5" + checksumTypeXxhash64 checksumType = "x-amz-checksum-xxhash64" + checksumTypeXxhash3 checksumType = "x-amz-checksum-xxhash3" + checksumTypeXxhash128 checksumType = "x-amz-checksum-xxhash128" ) func (c checksumType) isValid() bool { @@ -86,7 +91,12 @@ func (c checksumType) isValid() bool { c == checksumTypeCrc32c || c == checksumTypeSha1 || c == checksumTypeSha256 || - c == checksumTypeCrc64nvme + c == checksumTypeCrc64nvme || + c == checksumTypeSha512 || + c == checksumTypeMd5 || + c == checksumTypeXxhash64 || + c == checksumTypeXxhash3 || + c == checksumTypeXxhash128 } // Extracts and validates the checksum type from the 'X-Amz-Trailer' header diff --git a/s3api/utils/csum-reader.go b/s3api/utils/csum-reader.go index 2c5233c6..d34efd5e 100644 --- a/s3api/utils/csum-reader.go +++ b/s3api/utils/csum-reader.go @@ -18,6 +18,7 @@ import ( "crypto/md5" "crypto/sha1" "crypto/sha256" + "crypto/sha512" "encoding/base64" "encoding/hex" "errors" @@ -28,14 +29,18 @@ import ( "io" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/cespare/xxhash/v2" "github.com/versity/versitygw/s3err" + "github.com/zeebo/xxh3" ) // HashType identifies the checksum algorithm to be used type HashType string const ( - // HashTypeMd5 generates MD5 checksum for the data stream + // HashTypeContentMD5 generates MD5 checksum for the Content-MD5 header. + HashTypeContentMD5 HashType = "content-md5" + // HashTypeMd5 generates MD5 Base64-Encoded checksum for the data stream HashTypeMd5 HashType = "md5" // HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream HashTypeSha256 HashType = "sha256" @@ -43,12 +48,20 @@ const ( HashTypeSha256Hex HashType = "sha256-hex" // HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream HashTypeSha1 HashType = "sha1" + // HashTypeSha512 generates SHA512 Base64-Encoded checksum for the data stream + HashTypeSha512 HashType = "sha512" // HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream HashTypeCRC32 HashType = "crc32" // HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream HashTypeCRC32C HashType = "crc32c" // HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream HashTypeCRC64NVME HashType = "crc64nvme" + // HashTypeXXHASH64 generates XXHASH64 Base64-Encoded checksum for the data stream + HashTypeXXHASH64 HashType = "xxhash64" + // HashTypeXXHASH3 generates XXHASH3 Base64-Encoded checksum for the data stream + HashTypeXXHASH3 HashType = "xxhash3" + // HashTypeXXHASH128 generates XXHASH128 Base64-Encoded checksum for the data stream + HashTypeXXHASH128 HashType = "xxhash128" // HashTypeNone is a no-op checksum for the data stream HashTypeNone HashType = "none" ) @@ -75,7 +88,7 @@ var ( func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) { var hash hash.Hash switch ht { - case HashTypeMd5: + case HashTypeContentMD5, HashTypeMd5: hash = md5.New() case HashTypeSha256Hex: hash = sha256.New() @@ -83,12 +96,20 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e hash = sha256.New() case HashTypeSha1: hash = sha1.New() + case HashTypeSha512: + hash = sha512.New() case HashTypeCRC32: hash = crc32.NewIEEE() case HashTypeCRC32C: hash = crc32.New(crc32.MakeTable(crc32.Castagnoli)) case HashTypeCRC64NVME: hash = crc64.New(crc64NVMETable) + case HashTypeXXHASH64: + hash = xxhash.New() + case HashTypeXXHASH3: + hash = xxh3.New() + case HashTypeXXHASH128: + hash = xxh3.New128() case HashTypeNone: hash = noop{} default: @@ -112,11 +133,16 @@ func (hr *HashReader) Read(p []byte) (int, error) { } if errors.Is(readerr, io.EOF) && hr.sum != "" { switch hr.hashType { - case HashTypeMd5: + case HashTypeContentMD5: sum := hr.Sum() if sum != hr.sum { return n, s3err.GetAPIError(s3err.ErrBadDigest) } + case HashTypeMd5: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmMd5) + } case HashTypeSha256Hex: sum := hr.Sum() if sum != hr.sum { @@ -142,11 +168,31 @@ func (hr *HashReader) Read(p []byte) (int, error) { if sum != hr.sum { return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256) } + case HashTypeSha512: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha512) + } case HashTypeCRC64NVME: sum := hr.Sum() if sum != hr.sum { return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme) } + case HashTypeXXHASH64: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash64) + } + case HashTypeXXHASH3: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash3) + } + case HashTypeXXHASH128: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmXxhash128) + } default: return n, errInvalidHashType } @@ -161,7 +207,7 @@ func (hr *HashReader) SetReader(r io.Reader) { // Sum returns the checksum hash of the data read so far func (hr *HashReader) Sum() string { switch hr.hashType { - case HashTypeMd5: + case HashTypeContentMD5, HashTypeMd5: return Base64SumString(hr.hash.Sum(nil)) case HashTypeSha256Hex: return hex.EncodeToString(hr.hash.Sum(nil)) @@ -173,8 +219,16 @@ func (hr *HashReader) Sum() string { return Base64SumString(hr.hash.Sum(nil)) case HashTypeSha256: return Base64SumString(hr.hash.Sum(nil)) + case HashTypeSha512: + return Base64SumString(hr.hash.Sum(nil)) case HashTypeCRC64NVME: return Base64SumString(hr.hash.Sum(nil)) + case HashTypeXXHASH64: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeXXHASH3: + return Base64SumString(hr.hash.Sum(nil)) + case HashTypeXXHASH128: + return Base64SumString(hr.hash.Sum(nil)) default: return "" } @@ -197,8 +251,8 @@ func (n noop) Reset() {} func (n noop) Size() int { return 0 } func (n noop) BlockSize() int { return 1 } -// IsChecksumComposable tests if the final foll object crc can be calculated -// based on the part crc values. +// IsChecksumComposable tests if the final full-object crc can be calculated +// from the part crc values. func IsChecksumComposable(algo types.ChecksumAlgorithm) bool { switch algo { case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c, types.ChecksumAlgorithmCrc64nvme: @@ -309,17 +363,32 @@ func AddCRCChecksum(algo types.ChecksumAlgorithm, crc, partCrc string, partLen i // - CRC32C // - SHA1 // - SHA256 +// - SHA512 +// - MD5 +// - XXHASH64 +// - XXHASH3 +// - XXHASH128 func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) { var hasher hash.Hash switch ht { + case HashTypeMd5: + hasher = md5.New() case HashTypeSha256: hasher = sha256.New() case HashTypeSha1: hasher = sha1.New() + case HashTypeSha512: + hasher = sha512.New() case HashTypeCRC32: hasher = crc32.NewIEEE() case HashTypeCRC32C: hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case HashTypeXXHASH64: + hasher = xxhash.New() + case HashTypeXXHASH3: + hasher = xxh3.New() + case HashTypeXXHASH128: + hasher = xxh3.New128() case HashTypeNone: hasher = noop{} default: diff --git a/s3api/utils/csum-reader_test.go b/s3api/utils/csum-reader_test.go index e45e9817..89ee1e4d 100644 --- a/s3api/utils/csum-reader_test.go +++ b/s3api/utils/csum-reader_test.go @@ -15,12 +15,22 @@ package utils import ( + "bytes" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" "encoding/base64" + "hash" "hash/crc32" "hash/crc64" + "io" "testing" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/cespare/xxhash/v2" + "github.com/stretchr/testify/assert" + "github.com/zeebo/xxh3" ) func TestAddCRCChecksum_CRC32(t *testing.T) { @@ -118,3 +128,92 @@ func TestAddCRCChecksum_CRC64NVME(t *testing.T) { t.Errorf("CRC64NVME combine mismatch: got %x, want %x", combinedVal, crcFull) } } + +func base64HashForTest(h hash.Hash, data []byte) string { + h.Write(data) + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} + +func TestNewHashReader_NewChecksumAlgorithms(t *testing.T) { + data := []byte("checksum payload") + + tests := []struct { + name string + hashType HashType + hasher hash.Hash + }{ + {name: "md5", hashType: HashTypeMd5, hasher: md5.New()}, + {name: "sha1", hashType: HashTypeSha1, hasher: sha1.New()}, + {name: "sha256", hashType: HashTypeSha256, hasher: sha256.New()}, + {name: "sha512", hashType: HashTypeSha512, hasher: sha512.New()}, + {name: "crc32", hashType: HashTypeCRC32, hasher: crc32.NewIEEE()}, + {name: "crc32c", hashType: HashTypeCRC32C, hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))}, + {name: "crc64nvme", hashType: HashTypeCRC64NVME, hasher: crc64.New(crc64NVMETable)}, + {name: "xxhash64", hashType: HashTypeXXHASH64, hasher: xxhash.New()}, + {name: "xxhash3", hashType: HashTypeXXHASH3, hasher: xxh3.New()}, + {name: "xxhash128", hashType: HashTypeXXHASH128, hasher: xxh3.New128()}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expected := base64HashForTest(tt.hasher, data) + rdr, err := NewHashReader(bytes.NewReader(data), expected, tt.hashType) + if !assert.NoError(t, err) { + return + } + _, err = io.Copy(io.Discard, rdr) + assert.NoError(t, err) + assert.Equal(t, expected, rdr.Sum()) + }) + } +} + +func TestNewCompositeChecksumReader_NewChecksumAlgorithms(t *testing.T) { + part1 := []byte("part one") + part2 := []byte("part two") + + tests := []struct { + name string + hashType HashType + newHasher func() hash.Hash + }{ + {name: "md5", hashType: HashTypeMd5, newHasher: md5.New}, + {name: "sha1", hashType: HashTypeSha1, newHasher: sha1.New}, + {name: "sha256", hashType: HashTypeSha256, newHasher: sha256.New}, + {name: "sha512", hashType: HashTypeSha512, newHasher: sha512.New}, + {name: "crc32", hashType: HashTypeCRC32, newHasher: func() hash.Hash { return crc32.NewIEEE() }}, + {name: "crc32c", hashType: HashTypeCRC32C, newHasher: func() hash.Hash { return crc32.New(crc32.MakeTable(crc32.Castagnoli)) }}, + {name: "xxhash64", hashType: HashTypeXXHASH64, newHasher: func() hash.Hash { return xxhash.New() }}, + {name: "xxhash3", hashType: HashTypeXXHASH3, newHasher: func() hash.Hash { return xxh3.New() }}, + {name: "xxhash128", hashType: HashTypeXXHASH128, newHasher: func() hash.Hash { return xxh3.New128() }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + part1Sum := base64HashForTest(tt.newHasher(), part1) + part2Sum := base64HashForTest(tt.newHasher(), part2) + + composite, err := NewCompositeChecksumReader(tt.hashType) + if !assert.NoError(t, err) { + return + } + assert.NoError(t, composite.Process(part1Sum)) + assert.NoError(t, composite.Process(part2Sum)) + + part1Bytes, err := base64.StdEncoding.DecodeString(part1Sum) + if !assert.NoError(t, err) { + return + } + part2Bytes, err := base64.StdEncoding.DecodeString(part2Sum) + if !assert.NoError(t, err) { + return + } + expectedHasher := tt.newHasher() + expectedHasher.Write(part1Bytes) + expectedHasher.Write(part2Bytes) + expected := base64.StdEncoding.EncodeToString(expectedHasher.Sum(nil)) + + assert.Equal(t, expected, composite.Sum()) + }) + } +} diff --git a/s3api/utils/unsigned-chunk-reader.go b/s3api/utils/unsigned-chunk-reader.go index be7b6d3b..abcbec98 100644 --- a/s3api/utils/unsigned-chunk-reader.go +++ b/s3api/utils/unsigned-chunk-reader.go @@ -17,8 +17,10 @@ package utils import ( "bufio" "bytes" + "crypto/md5" "crypto/sha1" "crypto/sha256" + "crypto/sha512" "encoding/base64" "errors" "hash" @@ -30,8 +32,10 @@ import ( "strings" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/cespare/xxhash/v2" "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/s3err" + "github.com/zeebo/xxh3" ) var ( @@ -353,6 +357,16 @@ func getHasher(ct checksumType) (hash.Hash, error) { return sha1.New(), nil case checksumTypeSha256: return sha256.New(), nil + case checksumTypeSha512: + return sha512.New(), nil + case checksumTypeMd5: + return md5.New(), nil + case checksumTypeXxhash64: + return xxhash.New(), nil + case checksumTypeXxhash3: + return xxh3.New(), nil + case checksumTypeXxhash128: + return xxh3.New128(), nil default: return nil, errors.New("unsupported checksum type") } diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 5e37bdc0..833f7297 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -721,8 +721,13 @@ var checksumLengths = map[types.ChecksumAlgorithm]int{ types.ChecksumAlgorithmCrc32: 4, types.ChecksumAlgorithmCrc32c: 4, types.ChecksumAlgorithmCrc64nvme: 8, + types.ChecksumAlgorithmMd5: 16, types.ChecksumAlgorithmSha1: 20, types.ChecksumAlgorithmSha256: 32, + types.ChecksumAlgorithmSha512: 64, + types.ChecksumAlgorithmXxhash64: 8, + types.ChecksumAlgorithmXxhash3: 8, + types.ChecksumAlgorithmXxhash128: 16, } func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool { @@ -753,7 +758,12 @@ func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error { alg != types.ChecksumAlgorithmCrc32c && alg != types.ChecksumAlgorithmSha1 && alg != types.ChecksumAlgorithmSha256 && - alg != types.ChecksumAlgorithmCrc64nvme { + alg != types.ChecksumAlgorithmCrc64nvme && + alg != types.ChecksumAlgorithmSha512 && + alg != types.ChecksumAlgorithmMd5 && + alg != types.ChecksumAlgorithmXxhash64 && + alg != types.ChecksumAlgorithmXxhash3 && + alg != types.ChecksumAlgorithmXxhash128 { debuglogger.Logf("invalid checksum algorithm: %v\n", alg) return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm) } @@ -799,6 +809,26 @@ var checksumMap checksumSchema = checksumSchema{ types.ChecksumTypeFullObject: struct{}{}, "": struct{}{}, }, + types.ChecksumAlgorithmSha512: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmMd5: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmXxhash64: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmXxhash3: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmXxhash128: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + "": struct{}{}, + }, // Both could be empty "": checksumTypeSchema{ "": struct{}{}, diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 3b167ad6..fd6aa45c 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -707,6 +707,41 @@ func TestIsChecksumAlgorithmValid(t *testing.T) { }, wantErr: false, }, + { + name: "sha512", + args: args{ + alg: types.ChecksumAlgorithmSha512, + }, + wantErr: false, + }, + { + name: "md5", + args: args{ + alg: types.ChecksumAlgorithmMd5, + }, + wantErr: false, + }, + { + name: "xxhash64", + args: args{ + alg: types.ChecksumAlgorithmXxhash64, + }, + wantErr: false, + }, + { + name: "xxhash3", + args: args{ + alg: types.ChecksumAlgorithmXxhash3, + }, + wantErr: false, + }, + { + name: "xxhash128", + args: args{ + alg: types.ChecksumAlgorithmXxhash128, + }, + wantErr: false, + }, { name: "invalid", args: args{ @@ -806,6 +841,102 @@ func TestIsValidChecksum(t *testing.T) { }, want: true, }, + { + name: "invalid-crc64nvme", + args: args{ + checksum: "ww2FVQ==", + algorithm: types.ChecksumAlgorithmCrc64nvme, + }, + want: false, + }, + { + name: "valid-crc64nvme", + args: args{ + checksum: "AAAAAAAAAAA=", + algorithm: types.ChecksumAlgorithmCrc64nvme, + }, + want: true, + }, + { + name: "invalid-sha512", + args: args{ + checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=", + algorithm: types.ChecksumAlgorithmSha512, + }, + want: false, + }, + { + name: "valid-sha512", + args: args{ + checksum: "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==", + algorithm: types.ChecksumAlgorithmSha512, + }, + want: true, + }, + { + name: "invalid-md5", + args: args{ + checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=", + algorithm: types.ChecksumAlgorithmMd5, + }, + want: false, + }, + { + name: "valid-md5", + args: args{ + checksum: "1B2M2Y8AsgTpgAmY7PhCfg==", + algorithm: types.ChecksumAlgorithmMd5, + }, + want: true, + }, + { + name: "invalid-xxhash64", + args: args{ + checksum: "1B2M2Y8AsgTpgAmY7PhCfg==", + algorithm: types.ChecksumAlgorithmXxhash64, + }, + want: false, + }, + { + name: "valid-xxhash64", + args: args{ + checksum: "70bbN1HY6Zk=", + algorithm: types.ChecksumAlgorithmXxhash64, + }, + want: true, + }, + { + name: "invalid-xxhash3", + args: args{ + checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=", + algorithm: types.ChecksumAlgorithmXxhash3, + }, + want: false, + }, + { + name: "valid-xxhash3", + args: args{ + checksum: "LQaABTjTlMI=", + algorithm: types.ChecksumAlgorithmXxhash3, + }, + want: true, + }, + { + name: "invalid-xxhash128", + args: args{ + checksum: "70bbN1HY6Zk=", + algorithm: types.ChecksumAlgorithmXxhash128, + }, + want: false, + }, + { + name: "valid-xxhash128", + args: args{ + checksum: "maoG0wFHmNhgAcMkRo1Jfw==", + algorithm: types.ChecksumAlgorithmXxhash128, + }, + want: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1090,6 +1221,86 @@ func Test_checkChecksumTypeAndAlgo(t *testing.T) { }, wantErr: true, }, + { + name: "full_object-sha512", + args: args{ + algo: types.ChecksumAlgorithmSha512, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-md5", + args: args{ + algo: types.ChecksumAlgorithmMd5, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-xxhash64", + args: args{ + algo: types.ChecksumAlgorithmXxhash64, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-xxhash3", + args: args{ + algo: types.ChecksumAlgorithmXxhash3, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-xxhash128", + args: args{ + algo: types.ChecksumAlgorithmXxhash128, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "composite-sha512", + args: args{ + algo: types.ChecksumAlgorithmSha512, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-md5", + args: args{ + algo: types.ChecksumAlgorithmMd5, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-xxhash64", + args: args{ + algo: types.ChecksumAlgorithmXxhash64, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-xxhash3", + args: args{ + algo: types.ChecksumAlgorithmXxhash3, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-xxhash128", + args: args{ + algo: types.ChecksumAlgorithmXxhash128, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, { name: "composite-empty", args: args{ diff --git a/s3err/s3err.go b/s3err/s3err.go index ae9de4fd..676faead 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -763,7 +763,7 @@ var errorCodeResponse = map[ErrorCode]APIError{ }, ErrInvalidChecksumAlgorithm: { Code: "InvalidRequest", - Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]", + Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, CRC64NVME, MD5, SHA1, SHA256, SHA512, XXHASH128, XXHASH3, XXHASH64]", HTTPStatusCode: http.StatusBadRequest, }, ErrInvalidChecksumPart: { diff --git a/s3response/s3response.go b/s3response/s3response.go index 9525d5fe..5df79d93 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -37,6 +37,11 @@ type PutObjectOutput struct { ChecksumSHA1 *string ChecksumSHA256 *string ChecksumCRC64NVME *string + ChecksumSHA512 *string + ChecksumMD5 *string + ChecksumXXHASH64 *string + ChecksumXXHASH3 *string + ChecksumXXHASH128 *string Size *int64 ChecksumType types.ChecksumType } @@ -52,6 +57,11 @@ type Part struct { ChecksumSHA1 *string ChecksumSHA256 *string ChecksumCRC64NVME *string + ChecksumSHA512 *string + ChecksumMD5 *string + ChecksumXXHASH64 *string + ChecksumXXHASH3 *string + ChecksumXXHASH128 *string } func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -375,6 +385,11 @@ type CopyObjectResult struct { ChecksumCRC64NVME *string ChecksumSHA1 *string ChecksumSHA256 *string + ChecksumSHA512 *string + ChecksumMD5 *string + ChecksumXXHASH64 *string + ChecksumXXHASH3 *string + ChecksumXXHASH128 *string ChecksumType types.ChecksumType ETag *string LastModified *time.Time @@ -404,6 +419,11 @@ type CopyPartResult struct { ChecksumSHA1 *string ChecksumSHA256 *string ChecksumCRC64NVME *string + ChecksumSHA512 *string + ChecksumMD5 *string + ChecksumXXHASH64 *string + ChecksumXXHASH3 *string + ChecksumXXHASH128 *string // not included in the body CopySourceVersionId string `xml:"-"` @@ -439,6 +459,11 @@ type CompleteMultipartUploadResult struct { ChecksumSHA1 *string ChecksumSHA256 *string ChecksumCRC64NVME *string + ChecksumSHA512 *string + ChecksumMD5 *string + ChecksumXXHASH64 *string + ChecksumXXHASH3 *string + ChecksumXXHASH128 *string ChecksumType *types.ChecksumType } @@ -559,6 +584,11 @@ type PutObjectInput struct { ChecksumSHA1 *string ChecksumSHA256 *string ChecksumCRC64NVME *string + ChecksumSHA512 *string + ChecksumMD5 *string + ChecksumXXHASH64 *string + ChecksumXXHASH3 *string + ChecksumXXHASH128 *string ContentMD5 *string ExpectedBucketOwner *string GrantFullControl *string @@ -731,6 +761,11 @@ type Checksum struct { SHA1 *string SHA256 *string CRC64NVME *string + SHA512 *string + MD5 *string + XXHASH64 *string + XXHASH3 *string + XXHASH128 *string } // LocationConstraint represents the GetBucketLocation response diff --git a/tests/integration/CompleteMultipartUpload.go b/tests/integration/CompleteMultipartUpload.go index 78b621ab..625b20be 100644 --- a/tests/integration/CompleteMultipartUpload.go +++ b/tests/integration/CompleteMultipartUpload.go @@ -518,6 +518,26 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error { algo: types.ChecksumAlgorithmCrc64nvme, t: types.ChecksumTypeFullObject, }, + { + algo: types.ChecksumAlgorithmSha512, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmMd5, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmXxhash64, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmXxhash3, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmXxhash128, + t: types.ChecksumTypeComposite, + }, } { mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), @@ -534,15 +554,7 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error { cParts := []types.CompletedPart{} for _, el := range parts { - cParts = append(cParts, types.CompletedPart{ - ETag: el.ETag, - PartNumber: el.PartNumber, - ChecksumCRC32: el.ChecksumCRC32C, - ChecksumCRC32C: el.ChecksumCRC32C, - ChecksumSHA1: el.ChecksumSHA1, - ChecksumSHA256: el.ChecksumSHA256, - ChecksumCRC64NVME: el.ChecksumCRC64NVME, - }) + cParts = append(cParts, completedPartFromPart(el)) } mpInput := &s3.CompleteMultipartUploadInput{ @@ -566,6 +578,16 @@ func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error { mpInput.ChecksumSHA256 = getPtr("invalid_sha256") case types.ChecksumAlgorithmCrc64nvme: mpInput.ChecksumCRC64NVME = getPtr("invalid_crc64nvme") + case types.ChecksumAlgorithmSha512: + mpInput.ChecksumSHA512 = getPtr("invalid_sha512") + case types.ChecksumAlgorithmMd5: + mpInput.ChecksumMD5 = getPtr("invalid_md5") + case types.ChecksumAlgorithmXxhash64: + mpInput.ChecksumXXHASH64 = getPtr("invalid_xxhash64") + case types.ChecksumAlgorithmXxhash3: + mpInput.ChecksumXXHASH3 = getPtr("invalid_xxhash3") + case types.ChecksumAlgorithmXxhash128: + mpInput.ChecksumXXHASH128 = getPtr("invalid_xxhash128") } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) @@ -608,6 +630,26 @@ func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error { algo: types.ChecksumAlgorithmCrc64nvme, t: types.ChecksumTypeFullObject, }, + { + algo: types.ChecksumAlgorithmSha512, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmMd5, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmXxhash64, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmXxhash3, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmXxhash128, + t: types.ChecksumTypeComposite, + }, } { mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), withChecksumType(el.t)) @@ -623,34 +665,23 @@ func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error { cParts := []types.CompletedPart{} for _, el := range parts { - cParts = append(cParts, types.CompletedPart{ - ETag: el.ETag, - PartNumber: el.PartNumber, - ChecksumCRC32: el.ChecksumCRC32, - ChecksumCRC32C: el.ChecksumCRC32C, - ChecksumSHA1: el.ChecksumSHA1, - ChecksumSHA256: el.ChecksumSHA256, - ChecksumCRC64NVME: el.ChecksumCRC64NVME, - }) + cParts = append(cParts, completedPartFromPart(el)) } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + input := &s3.CompleteMultipartUploadInput{ Bucket: &bucket, Key: &obj, UploadId: mp.UploadId, MultipartUpload: &types.CompletedMultipartUpload{ Parts: cParts, }, - // Provide one of the parts checksum. In any case - // the final checksum will differ from one of the parts checksum - ChecksumCRC32: cParts[0].ChecksumCRC32, - ChecksumCRC32C: cParts[0].ChecksumCRC32C, - ChecksumSHA1: cParts[0].ChecksumSHA1, - ChecksumSHA256: cParts[0].ChecksumSHA256, - ChecksumCRC64NVME: cParts[0].ChecksumCRC64NVME, - ChecksumType: el.t, - }) + ChecksumType: el.t, + } + // Provide one of the part checksums. The final checksum will differ + // from a single part checksum. + setCompleteMultipartUploadChecksum(input, el.algo, getCompletedPartChecksum(cParts[0], el.algo)) + _, err = s3client.CompleteMultipartUpload(ctx, input) cancel() if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { return err @@ -848,6 +879,11 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e types.ChecksumAlgorithmCrc32c, types.ChecksumAlgorithmSha1, types.ChecksumAlgorithmSha256, + types.ChecksumAlgorithmSha512, + types.ChecksumAlgorithmMd5, + types.ChecksumAlgorithmXxhash64, + types.ChecksumAlgorithmXxhash3, + types.ChecksumAlgorithmXxhash128, } { mp, err := createMp(s3client, bucket, obj, withChecksumType(types.ChecksumTypeComposite), withChecksum(algo)) if err != nil { @@ -867,29 +903,13 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e completeParts := make([]types.CompletedPart, 0, len(parts)) for _, part := range parts { - switch algo { - case types.ChecksumAlgorithmCrc32: - err = processCompositeChecksum(hasher, getString(part.ChecksumCRC32)) - case types.ChecksumAlgorithmCrc32c: - err = processCompositeChecksum(hasher, getString(part.ChecksumCRC32C)) - case types.ChecksumAlgorithmSha1: - err = processCompositeChecksum(hasher, getString(part.ChecksumSHA1)) - case types.ChecksumAlgorithmSha256: - err = processCompositeChecksum(hasher, getString(part.ChecksumSHA256)) - } + err = processCompositeChecksum(hasher, getString(getPartChecksum(part, algo))) if err != nil { return fmt.Errorf("test %v failed: %s", i, err) } - completeParts = append(completeParts, types.CompletedPart{ - ETag: part.ETag, - PartNumber: part.PartNumber, - ChecksumCRC32: part.ChecksumCRC32, - ChecksumCRC32C: part.ChecksumCRC32C, - ChecksumSHA1: part.ChecksumSHA1, - ChecksumSHA256: part.ChecksumSHA256, - }) + completeParts = append(completeParts, completedPartFromPart(part)) } checksum := fmt.Sprintf("%s-%v", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), len(parts)) @@ -903,16 +923,7 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e UploadId: mp.UploadId, } - switch algo { - case types.ChecksumAlgorithmCrc32: - completeMpInput.ChecksumCRC32 = &checksum - case types.ChecksumAlgorithmCrc32c: - completeMpInput.ChecksumCRC32C = &checksum - case types.ChecksumAlgorithmSha1: - completeMpInput.ChecksumSHA1 = &checksum - case types.ChecksumAlgorithmSha256: - completeMpInput.ChecksumSHA256 = &checksum - } + setCompleteMultipartUploadChecksum(completeMpInput, algo, &checksum) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.CompleteMultipartUpload(ctx, completeMpInput) @@ -921,17 +932,7 @@ func CompleteMultipartUpload_should_verify_final_composite_checksum(s *S3Conf) e return fmt.Errorf("test %v failed: %s", i, err) } - var gotSum string - switch algo { - case types.ChecksumAlgorithmCrc32: - gotSum = getString(res.ChecksumCRC32) - case types.ChecksumAlgorithmCrc32c: - gotSum = getString(res.ChecksumCRC32C) - case types.ChecksumAlgorithmSha1: - gotSum = getString(res.ChecksumSHA1) - case types.ChecksumAlgorithmSha256: - gotSum = getString(res.ChecksumSHA256) - } + gotSum := getString(getCompleteMultipartUploadChecksum(res, algo)) if gotSum != checksum { return fmt.Errorf("test %v failed: expected the final checksum to be %s, instead got %s", i, checksum, gotSum) @@ -947,20 +948,27 @@ func CompleteMultipartUpload_invalid_final_composite_checksum(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" for i, test := range []struct { - algo types.ChecksumAlgorithm - crc32 *string - crc32c *string - sha1 *string - sha256 *string + algo types.ChecksumAlgorithm + checksum *string }{ - {types.ChecksumAlgorithmCrc32, getPtr("invalid_checksum"), nil, nil, nil}, - {types.ChecksumAlgorithmCrc32, getPtr("ImIEBA==-smth"), nil, nil, nil}, - {types.ChecksumAlgorithmCrc32c, nil, getPtr("invalid_checksum"), nil, nil}, - {types.ChecksumAlgorithmCrc32c, nil, getPtr("AQIDBA==-12a"), nil, nil}, - {types.ChecksumAlgorithmSha1, nil, nil, getPtr("invalid_checksum"), nil}, - {types.ChecksumAlgorithmSha1, nil, nil, getPtr("2jmj7l5rSw0yVb/vlWAYkK/YBwk=-10-20"), nil}, - {types.ChecksumAlgorithmSha256, nil, nil, nil, getPtr("invalid_checksum")}, - {types.ChecksumAlgorithmSha256, nil, nil, nil, getPtr("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=--3")}, + {types.ChecksumAlgorithmCrc32, getPtr("invalid_checksum")}, + {types.ChecksumAlgorithmCrc32, getPtr("ImIEBA==-smth")}, + {types.ChecksumAlgorithmCrc32c, getPtr("invalid_checksum")}, + {types.ChecksumAlgorithmCrc32c, getPtr("AQIDBA==-12a")}, + {types.ChecksumAlgorithmSha1, getPtr("invalid_checksum")}, + {types.ChecksumAlgorithmSha1, getPtr("2jmj7l5rSw0yVb/vlWAYkK/YBwk=-10-20")}, + {types.ChecksumAlgorithmSha256, getPtr("invalid_checksum")}, + {types.ChecksumAlgorithmSha256, getPtr("47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=--3")}, + {types.ChecksumAlgorithmSha512, getPtr("invalid_checksum")}, + {types.ChecksumAlgorithmSha512, getPtr("z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==-bad")}, + {types.ChecksumAlgorithmMd5, getPtr("invalid_checksum")}, + {types.ChecksumAlgorithmMd5, getPtr("1B2M2Y8AsgTpgAmY7PhCfg==-bad")}, + {types.ChecksumAlgorithmXxhash64, getPtr("invalid_checksum")}, + {types.ChecksumAlgorithmXxhash64, getPtr("70bbN1HY6Zk=-bad")}, + {types.ChecksumAlgorithmXxhash3, getPtr("invalid_checksum")}, + {types.ChecksumAlgorithmXxhash3, getPtr("LQaABTjTlMI=-bad")}, + {types.ChecksumAlgorithmXxhash128, getPtr("invalid_checksum")}, + {types.ChecksumAlgorithmXxhash128, getPtr("maoG0wFHmNhgAcMkRo1Jfw==-bad")}, } { mp, err := createMp(s3client, bucket, obj, withChecksum(test.algo), withChecksumType(types.ChecksumTypeComposite)) if err != nil { @@ -975,29 +983,20 @@ func CompleteMultipartUpload_invalid_final_composite_checksum(s *S3Conf) error { completeParts := make([]types.CompletedPart, 0, len(parts)) for _, part := range parts { - completeParts = append(completeParts, types.CompletedPart{ - ETag: part.ETag, - PartNumber: part.PartNumber, - ChecksumCRC32: part.ChecksumCRC32, - ChecksumCRC32C: part.ChecksumCRC32C, - ChecksumSHA1: part.ChecksumSHA1, - ChecksumSHA256: part.ChecksumSHA256, - }) + completeParts = append(completeParts, completedPartFromPart(part)) } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + input := &s3.CompleteMultipartUploadInput{ Bucket: &bucket, Key: &obj, UploadId: mp.UploadId, MultipartUpload: &types.CompletedMultipartUpload{ Parts: completeParts, }, - ChecksumCRC32: test.crc32, - ChecksumCRC32C: test.crc32c, - ChecksumSHA1: test.sha1, - ChecksumSHA256: test.sha256, - }) + } + setCompleteMultipartUploadChecksum(input, test.algo, test.checksum) + _, err = s3client.CompleteMultipartUpload(ctx, input) cancel() if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(test.algo))))); err != nil { return fmt.Errorf("test %v failed: %w", i, err) @@ -1090,21 +1089,20 @@ func CompleteMultipartUpload_should_ignore_the_final_checksum(s *S3Conf) error { return err } - if res.ChecksumCRC32 != nil { - return fmt.Errorf("expected nil crc32 checksum, insted got %v", - *res.ChecksumCRC32) - } - if res.ChecksumCRC32C != nil { - return fmt.Errorf("expected nil crc32c checksum, insted got %v", - *res.ChecksumCRC32C) - } - if res.ChecksumSHA1 != nil { - return fmt.Errorf("expected nil sha1 checksum, insted got %v", - *res.ChecksumSHA1) - } - if res.ChecksumSHA256 != nil { - return fmt.Errorf("expected nil sha256 checksum, insted got %v", - *res.ChecksumSHA256) + for _, algo := range []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmCrc32, + types.ChecksumAlgorithmCrc32c, + types.ChecksumAlgorithmSha1, + types.ChecksumAlgorithmSha256, + types.ChecksumAlgorithmSha512, + types.ChecksumAlgorithmMd5, + types.ChecksumAlgorithmXxhash64, + types.ChecksumAlgorithmXxhash3, + types.ChecksumAlgorithmXxhash128, + } { + if checksum := getCompleteMultipartUploadChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, insted got %v", algo, *checksum) + } } // If no checksum is specified on mp creation, it should default // to crc64nvme diff --git a/tests/integration/CopyObject.go b/tests/integration/CopyObject.go index 4de9eb38..f2eae7f7 100644 --- a/tests/integration/CopyObject.go +++ b/tests/integration/CopyObject.go @@ -1461,17 +1461,13 @@ func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error { if out.CopyObjectResult.ChecksumCRC32 == nil { return fmt.Errorf("expected non empty crc32 checksum") } - if out.CopyObjectResult.ChecksumCRC32C != nil { - return fmt.Errorf("expected empty crc32c checksum") - } - if out.CopyObjectResult.ChecksumSHA1 != nil { - return fmt.Errorf("expected empty sha1 checksum") - } - if out.CopyObjectResult.ChecksumSHA256 != nil { - return fmt.Errorf("expected empty sha256 checksum") - } - if out.CopyObjectResult.ChecksumCRC64NVME != nil { - return fmt.Errorf("expected empty crc64nvme checksum") + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if algo == types.ChecksumAlgorithmCrc32 { + continue + } + if checksum := getCopyObjectChecksum(out.CopyObjectResult, algo); checksum != nil { + return fmt.Errorf("expected empty %s checksum", algo) + } } ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) @@ -1488,17 +1484,13 @@ func CopyObject_to_itself_by_replacing_the_checksum(s *S3Conf) error { if res.ChecksumCRC32 == nil { return fmt.Errorf("expected non empty crc32 checksum") } - if res.ChecksumCRC32C != nil { - return fmt.Errorf("expected empty crc32c checksum") - } - if res.ChecksumSHA1 != nil { - return fmt.Errorf("expected empty sha1 checksum") - } - if res.ChecksumSHA256 != nil { - return fmt.Errorf("expected empty sha256 checksum") - } - if res.ChecksumCRC64NVME != nil { - return fmt.Errorf("expected empty crc64nvme checksum") + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if algo == types.ChecksumAlgorithmCrc32 { + continue + } + if checksum := getHeadObjectChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected empty %s checksum", algo) + } } return nil diff --git a/tests/integration/CreateMultipartUpload.go b/tests/integration/CreateMultipartUpload.go index b7b7c529..59b64340 100644 --- a/tests/integration/CreateMultipartUpload.go +++ b/tests/integration/CreateMultipartUpload.go @@ -390,6 +390,11 @@ func CreateMultipartUpload_type_algo_mismatch(s *S3Conf) error { {types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc64nvme}, {types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha1}, {types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha256}, + {types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha512}, + {types.ChecksumTypeFullObject, types.ChecksumAlgorithmMd5}, + {types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash64}, + {types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash3}, + {types.ChecksumTypeFullObject, types.ChecksumAlgorithmXxhash128}, } { _, err := createMp(s3client, bucket, "my-obj", withChecksum(test.algo), withChecksumType(test.chType)) if err := checkApiErr(err, s3err.GetChecksumSchemaMismatchErr(test.algo, test.chType)); err != nil { @@ -414,6 +419,11 @@ func CreateMultipartUpload_valid_algo_type(s *S3Conf) error { {types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c}, {types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1}, {types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmSha512}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmMd5}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash64}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash3}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash128}, // full object type {types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc64nvme}, {types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc32}, diff --git a/tests/integration/GetObject.go b/tests/integration/GetObject.go index 71b3cd6f..7df37b39 100644 --- a/tests/integration/GetObject.go +++ b/tests/integration/GetObject.go @@ -233,38 +233,13 @@ func GetObject_invalid_parent(s *S3Conf) error { func GetObject_checksums(s *S3Conf) error { testName := "GetObject_checksums" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - objs := []struct { - key string - checksumAlgo types.ChecksumAlgorithm - }{ - { - key: "obj-1", - checksumAlgo: types.ChecksumAlgorithmCrc32, - }, - { - key: "obj-2", - checksumAlgo: types.ChecksumAlgorithmCrc32c, - }, - { - key: "obj-3", - checksumAlgo: types.ChecksumAlgorithmSha1, - }, - { - key: "obj-4", - checksumAlgo: types.ChecksumAlgorithmSha256, - }, - { - key: "obj-5", - checksumAlgo: types.ChecksumAlgorithmCrc64nvme, - }, - } - - for i, el := range objs { - out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{ + for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() { + key := fmt.Sprintf("obj-%v", i+1) + out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{ Bucket: &bucket, - Key: &el.key, - ChecksumAlgorithm: el.checksumAlgo, - }, s3client) + Key: &key, + ChecksumAlgorithm: checksumAlgo, + }, s3client, withPutObjectChecksumAlgo(checksumAlgo)) if err != nil { return err } @@ -272,7 +247,7 @@ func GetObject_checksums(s *S3Conf) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ Bucket: &bucket, - Key: &el.key, + Key: &key, ChecksumMode: types.ChecksumModeEnabled, }) cancel() @@ -281,28 +256,11 @@ func GetObject_checksums(s *S3Conf) error { } if res.ChecksumType != types.ChecksumTypeFullObject { - return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", - el.key, types.ChecksumTypeFullObject, res.ChecksumType) + return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v", + key, types.ChecksumTypeFullObject, res.ChecksumType) } - if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { - return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", - getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32)) - } - if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { - return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", - getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C)) - } - if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) { - return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", - getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1)) - } - if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { - return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", - getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256)) - } - if getString(res.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) { - return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v", - getString(out.res.ChecksumCRC64NVME), getString(res.ChecksumCRC64NVME)) + if got, expected := getString(getGetObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected { + return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got) } } @@ -313,43 +271,31 @@ func GetObject_checksums(s *S3Conf) error { func GetObject_dir_object_checksum(s *S3Conf) error { testName := "GetObject_dir_object_checksum" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - for i, obj := range []struct { - key string - expectedSum string + for i, test := range []struct { checksumAlgo types.ChecksumAlgorithm + expectedSum string }{ - { - key: "obj-1/", - expectedSum: "AAAAAA==", - checksumAlgo: types.ChecksumAlgorithmCrc32, - }, - { - key: "obj-2/", - expectedSum: "AAAAAA==", - checksumAlgo: types.ChecksumAlgorithmCrc32c, - }, - { - key: "obj-3/", - expectedSum: "AAAAAAAAAAA=", - checksumAlgo: types.ChecksumAlgorithmCrc64nvme, - }, - { - key: "obj-4/", - expectedSum: "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", - checksumAlgo: types.ChecksumAlgorithmSha1, - }, - { - key: "obj-5/", - expectedSum: "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", - checksumAlgo: types.ChecksumAlgorithmSha256, - }, + {types.ChecksumAlgorithmCrc32, "AAAAAA=="}, + {types.ChecksumAlgorithmCrc32c, "AAAAAA=="}, + {types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="}, + {types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="}, + {types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="}, + {types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="}, + {types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="}, + {types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="}, + {types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="}, + {types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="}, } { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + key := fmt.Sprintf("obj-%v/", i+1) + input := &s3.PutObjectInput{ Bucket: &bucket, - Key: &obj.key, - ChecksumAlgorithm: obj.checksumAlgo, - }) + Key: &key, + ChecksumAlgorithm: test.checksumAlgo, + } + setPutObjectChecksum(input, test.checksumAlgo, &test.expectedSum) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, input) cancel() if err != nil { return fmt.Errorf("test %v failed: %w", i+1, err) @@ -358,7 +304,7 @@ func GetObject_dir_object_checksum(s *S3Conf) error { ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ Bucket: &bucket, - Key: &obj.key, + Key: &key, ChecksumMode: types.ChecksumModeEnabled, }) cancel() @@ -368,25 +314,13 @@ func GetObject_dir_object_checksum(s *S3Conf) error { if res.ChecksumType != types.ChecksumTypeFullObject { return fmt.Errorf("test %v failed: expected the %v object checksum type to be %v, instaed got %v", - i+1, obj.key, types.ChecksumTypeFullObject, res.ChecksumType) + i+1, key, types.ChecksumTypeFullObject, res.ChecksumType) } - var gotSum *string - switch obj.checksumAlgo { - case types.ChecksumAlgorithmCrc32: - gotSum = res.ChecksumCRC32 - case types.ChecksumAlgorithmCrc32c: - gotSum = res.ChecksumCRC32C - case types.ChecksumAlgorithmCrc64nvme: - gotSum = res.ChecksumCRC64NVME - case types.ChecksumAlgorithmSha1: - gotSum = res.ChecksumSHA1 - case types.ChecksumAlgorithmSha256: - gotSum = res.ChecksumSHA256 - } + gotSum := getGetObjectChecksum(res, test.checksumAlgo) - if getString(gotSum) != obj.expectedSum { - return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, obj.checksumAlgo, obj.expectedSum, getString(gotSum)) + if getString(gotSum) != test.expectedSum { + return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, test.checksumAlgo, test.expectedSum, getString(gotSum)) } } @@ -885,20 +819,10 @@ func GetObject_not_enabled_checksum_mode(s *S3Conf) error { return err } - if res.ChecksumCRC32 != nil { - return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32) - } - if res.ChecksumCRC32C != nil { - return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C) - } - if res.ChecksumSHA1 != nil { - return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1) - } - if res.ChecksumSHA256 != nil { - return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256) - } - if res.ChecksumCRC64NVME != nil { - return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME) + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if checksum := getGetObjectChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum) + } } return nil @@ -1704,20 +1628,10 @@ func GetObject_ranged_with_checksum_mode(s *S3Conf) error { testName := "GetObject_ranged_with_checksum_mode" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { checkNoChecksums := func(res *s3.GetObjectOutput) error { - if res.ChecksumCRC32 != nil { - return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32) - } - if res.ChecksumCRC32C != nil { - return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C) - } - if res.ChecksumSHA1 != nil { - return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1) - } - if res.ChecksumSHA256 != nil { - return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256) - } - if res.ChecksumCRC64NVME != nil { - return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME) + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if checksum := getGetObjectChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum) + } } return nil } diff --git a/tests/integration/GetObjectAttributes.go b/tests/integration/GetObjectAttributes.go index d9aed904..bf087fcb 100644 --- a/tests/integration/GetObjectAttributes.go +++ b/tests/integration/GetObjectAttributes.go @@ -235,38 +235,13 @@ func GetObjectAttributes_existing_object(s *S3Conf) error { func GetObjectAttributes_checksums(s *S3Conf) error { testName := "GetObjectAttributes_checksums" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - objs := []struct { - key string - checksumAlgo types.ChecksumAlgorithm - }{ - { - key: "obj-1", - checksumAlgo: types.ChecksumAlgorithmCrc32, - }, - { - key: "obj-2", - checksumAlgo: types.ChecksumAlgorithmCrc32c, - }, - { - key: "obj-3", - checksumAlgo: types.ChecksumAlgorithmSha1, - }, - { - key: "obj-4", - checksumAlgo: types.ChecksumAlgorithmSha256, - }, - { - key: "obj-5", - checksumAlgo: types.ChecksumAlgorithmCrc64nvme, - }, - } - - for i, el := range objs { - out, err := putObjectWithData(int64(i*120), &s3.PutObjectInput{ + for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() { + key := fmt.Sprintf("obj-%v", i+1) + out, err := putObjectWithData(int64((i+1)*120), &s3.PutObjectInput{ Bucket: &bucket, - Key: &el.key, - ChecksumAlgorithm: el.checksumAlgo, - }, s3client) + Key: &key, + ChecksumAlgorithm: checksumAlgo, + }, s3client, withPutObjectChecksumAlgo(checksumAlgo)) if err != nil { return err } @@ -274,7 +249,7 @@ func GetObjectAttributes_checksums(s *S3Conf) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ Bucket: &bucket, - Key: &el.key, + Key: &key, ObjectAttributes: []types.ObjectAttributes{ types.ObjectAttributesChecksum, }, @@ -288,28 +263,11 @@ func GetObjectAttributes_checksums(s *S3Conf) error { return fmt.Errorf("expected non-nil checksum in the response") } if res.Checksum.ChecksumType != types.ChecksumTypeFullObject { - return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", - el.key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType) + return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v", + key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType) } - if getString(res.Checksum.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { - return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", - getString(out.res.ChecksumCRC32), getString(res.Checksum.ChecksumCRC32)) - } - if getString(res.Checksum.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { - return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", - getString(out.res.ChecksumCRC32C), getString(res.Checksum.ChecksumCRC32C)) - } - if getString(res.Checksum.ChecksumSHA1) != getString(out.res.ChecksumSHA1) { - return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", - getString(out.res.ChecksumSHA1), getString(res.Checksum.ChecksumSHA1)) - } - if getString(res.Checksum.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { - return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", - getString(out.res.ChecksumSHA256), getString(res.Checksum.ChecksumSHA256)) - } - if getString(res.Checksum.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) { - return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v", - getString(out.res.ChecksumCRC64NVME), getString(res.Checksum.ChecksumCRC64NVME)) + if got, expected := getString(getObjectAttributesChecksum(res.Checksum, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected { + return fmt.Errorf("expected %s checksum to be %s, instead got %s", checksumAlgo, expected, got) } } return nil diff --git a/tests/integration/HeadObject.go b/tests/integration/HeadObject.go index bbd6c997..b89a8e72 100644 --- a/tests/integration/HeadObject.go +++ b/tests/integration/HeadObject.go @@ -150,17 +150,10 @@ func HeadObject_not_enabled_checksum_mode(s *S3Conf) error { return err } - if res.ChecksumCRC32 != nil { - return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32) - } - if res.ChecksumCRC32C != nil { - return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C) - } - if res.ChecksumSHA1 != nil { - return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1) - } - if res.ChecksumSHA256 != nil { - return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256) + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if checksum := getHeadObjectChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum) + } } return nil @@ -170,38 +163,13 @@ func HeadObject_not_enabled_checksum_mode(s *S3Conf) error { func HeadObject_checksums(s *S3Conf) error { testName := "HeadObject_checksums" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - objs := []struct { - key string - checksumAlgo types.ChecksumAlgorithm - }{ - { - key: "obj-1", - checksumAlgo: types.ChecksumAlgorithmCrc32, - }, - { - key: "obj-2", - checksumAlgo: types.ChecksumAlgorithmCrc32c, - }, - { - key: "obj-3", - checksumAlgo: types.ChecksumAlgorithmSha1, - }, - { - key: "obj-4", - checksumAlgo: types.ChecksumAlgorithmSha256, - }, - { - key: "obj-5", - checksumAlgo: types.ChecksumAlgorithmCrc64nvme, - }, - } - - for i, el := range objs { - out, err := putObjectWithData(int64(i*200), &s3.PutObjectInput{ + for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() { + key := fmt.Sprintf("obj-%v", i+1) + out, err := putObjectWithData(int64((i+1)*200), &s3.PutObjectInput{ Bucket: &bucket, - Key: &el.key, - ChecksumAlgorithm: el.checksumAlgo, - }, s3client) + Key: &key, + ChecksumAlgorithm: checksumAlgo, + }, s3client, withPutObjectChecksumAlgo(checksumAlgo)) if err != nil { return err } @@ -209,7 +177,7 @@ func HeadObject_checksums(s *S3Conf) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: &bucket, - Key: &el.key, + Key: &key, ChecksumMode: types.ChecksumModeEnabled, }) cancel() @@ -218,22 +186,10 @@ func HeadObject_checksums(s *S3Conf) error { } if res.ChecksumType != types.ChecksumTypeFullObject { - return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.ChecksumType) + return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v", key, types.ChecksumTypeFullObject, res.ChecksumType) } - if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { - return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32)) - } - if getString(res.ChecksumCRC32C) != getString(out.res.ChecksumCRC32C) { - return fmt.Errorf("expected crc32c checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32C), getString(res.ChecksumCRC32C)) - } - if getString(res.ChecksumSHA1) != getString(out.res.ChecksumSHA1) { - return fmt.Errorf("expected sha1 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA1), getString(res.ChecksumSHA1)) - } - if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) { - return fmt.Errorf("expected sha256 checksum to be %v, instead got %v", getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256)) - } - if getString(res.ChecksumCRC64NVME) != getString(out.res.ChecksumCRC64NVME) { - return fmt.Errorf("expected crc64nvme checksum to be %v, instead got %v", getString(out.res.ChecksumCRC64NVME), getString(res.ChecksumCRC64NVME)) + if got, expected := getString(getHeadObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected { + return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got) } } @@ -1311,20 +1267,10 @@ func HeadObject_ranged_with_checksum_mode(s *S3Conf) error { testName := "HeadObject_ranged_with_checksum_mode" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { checkNoChecksums := func(res *s3.HeadObjectOutput) error { - if res.ChecksumCRC32 != nil { - return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32) - } - if res.ChecksumCRC32C != nil { - return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C) - } - if res.ChecksumSHA1 != nil { - return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1) - } - if res.ChecksumSHA256 != nil { - return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256) - } - if res.ChecksumCRC64NVME != nil { - return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", *res.ChecksumCRC64NVME) + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if checksum := getHeadObjectChecksum(res, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum) + } } return nil } diff --git a/tests/integration/ListMultipartUploads.go b/tests/integration/ListMultipartUploads.go index f60d946e..9bf8b267 100644 --- a/tests/integration/ListMultipartUploads.go +++ b/tests/integration/ListMultipartUploads.go @@ -612,6 +612,32 @@ func ListMultipartUploads_with_checksums(s *S3Conf) error { algo: types.ChecksumAlgorithmCrc64nvme, t: types.ChecksumTypeFullObject, }, + { + obj: "obj-6", + algo: types.ChecksumAlgorithmSha512, + t: types.ChecksumTypeComposite, + }, + { + obj: "obj-7", + algo: types.ChecksumAlgorithmMd5, + t: types.ChecksumTypeComposite, + }, + { + obj: "obj-8", + algo: types.ChecksumAlgorithmXxhash64, + t: types.ChecksumTypeComposite, + }, + { + obj: "obj-9", + algo: types.ChecksumAlgorithmXxhash3, + t: types.ChecksumTypeComposite, + }, + // obj-91 to preserve the lexicographical order + { + obj: "obj-91", + algo: types.ChecksumAlgorithmXxhash128, + t: types.ChecksumTypeComposite, + }, } { key := el.obj mp, err := createMp(s3client, bucket, key, withChecksum(el.algo), withChecksumType(el.t)) diff --git a/tests/integration/ListObjects.go b/tests/integration/ListObjects.go index 9902c04e..8e875f12 100644 --- a/tests/integration/ListObjects.go +++ b/tests/integration/ListObjects.go @@ -349,7 +349,7 @@ func ListObjects_with_checksum(s *S3Conf) error { Bucket: &bucket, Key: &key, ChecksumAlgorithm: el, - }, s3client) + }, s3client, withPutObjectChecksumAlgo(el)) if err != nil { return err } diff --git a/tests/integration/ListObjectsV2.go b/tests/integration/ListObjectsV2.go index beb8dd88..fad1023e 100644 --- a/tests/integration/ListObjectsV2.go +++ b/tests/integration/ListObjectsV2.go @@ -578,7 +578,7 @@ func ListObjectsV2_with_checksum(s *S3Conf) error { Bucket: &bucket, Key: &key, ChecksumAlgorithm: el, - }, s3client) + }, s3client, withPutObjectChecksumAlgo(el)) if err != nil { return err } diff --git a/tests/integration/PostObject.go b/tests/integration/PostObject.go index 5a7f9200..04a7c7e5 100644 --- a/tests/integration/PostObject.go +++ b/tests/integration/PostObject.go @@ -1070,6 +1070,11 @@ func PostObject_checksums_success(s *S3Conf) error { {"x-amz-checksum-crc64nvme", "rsrzr5yYqFU="}, {"x-amz-checksum-sha1", "9I3YU4IIYIFsddVND1hNyGMyenw="}, {"x-amz-checksum-sha256", "kW8AJ6V1B0znKjMXd8NHjWUT94alkb2JLaGld78jNfk="}, + {"x-amz-checksum-sha512", "Dh4h7PEF7IU9JNcohnrXBhPCFmOkaTB0sqNhnBvTnWa1iMM3I7tGbHJCToDjymPCSQeKs0e6uUKFAOfuQwWdDQ=="}, + {"x-amz-checksum-md5", "63M6AMDJ0zbmVpGjerVCkw=="}, + {"x-amz-checksum-xxhash64", "+lb36/ER8bo="}, + {"x-amz-checksum-xxhash3", "jw+pSh/pbMQ="}, + {"x-amz-checksum-xxhash128", "8BLDqqIWji+ITOsp/JjN/Q=="}, } { resp, err := sendPostObject(PostRequestConfig{ bucket: bucket, diff --git a/tests/integration/PutObject.go b/tests/integration/PutObject.go index e40c58ff..1a01c5aa 100644 --- a/tests/integration/PutObject.go +++ b/tests/integration/PutObject.go @@ -702,98 +702,31 @@ func PutObject_invalid_checksum_header(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - for i, el := range []struct { - algo string - crc32 *string - crc32c *string - sha1 *string - sha256 *string - crc64nvme *string - }{ - // CRC32 tests - { - algo: "crc32", - crc32: getPtr(""), - }, - { - algo: "crc32", - crc32: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "crc32", - crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32 - }, - // CRC32C tests - { - algo: "crc32c", - crc32c: getPtr(""), - }, - { - algo: "crc32c", - crc32c: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "crc32c", - crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c - }, - // SHA1 tests - { - algo: "sha1", - sha1: getPtr(""), - }, - { - algo: "sha1", - sha1: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "sha1", - sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1 - }, - // SHA256 tests - { - algo: "sha256", - sha256: getPtr(""), - }, - { - algo: "sha256", - sha256: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "sha256", - sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56 - }, - // CRC64Nvme tests - { - algo: "crc64nvme", - sha256: getPtr(""), - }, - { - algo: "crc64nvme", - sha256: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "crc64nvme", - sha256: getPtr("ZHNhZmRzYWZzZGFmZHNhZg=="), // valid base64 but not crc64nvme - }, - } { - _, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - ChecksumCRC32: el.crc32, - ChecksumCRC32C: el.crc32c, - ChecksumSHA1: el.sha1, - ChecksumSHA256: el.sha256, - ChecksumCRC64NVME: el.crc64nvme, - }, s3client) + i := 0 + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + // tests against: + // - empty string + // - invalid base64 + // - valid base64, but invalid checksum + for _, checksum := range []string{"", "invalid_base64!", "c2RhZnNhZGZzZGFm"} { + input := &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + } + setPutObjectChecksum(input, algo, getPtr(checksum)) - // FIXME: The error message for PutObject is not properly serialized by the sdk - // References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921 + _, err := putObjectWithData(int64((i+1)*100), input, s3client) + i++ - // if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { - // return err - // } - if err := checkSdkApiErr(err, "InvalidRequest"); err != nil { - return err + // FIXME: The error message for PutObject is not properly serialized by the sdk + // References to aws sdk issue https://github.com/aws/aws-sdk-go-v2/issues/2921 + + // if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + // return err + // } + if err := checkSdkApiErr(err, "InvalidRequest"); err != nil { + return err + } } } @@ -807,62 +740,33 @@ func PutObject_incorrect_checksums(s *S3Conf) error { obj := "my-obj" dirObj := "dir-object/" - for i, el := range []struct { - algo types.ChecksumAlgorithm - crc32 *string - crc32c *string - sha1 *string - sha256 *string - crc64nvme *string - }{ - { - algo: types.ChecksumAlgorithmCrc32, - crc32: getPtr("DUoRhQ=="), - }, - { - algo: types.ChecksumAlgorithmCrc32c, - crc32c: getPtr("yZRlqg=="), - }, - { - algo: types.ChecksumAlgorithmSha1, - sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), - }, - { - algo: types.ChecksumAlgorithmSha256, - sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="), - }, - { - algo: types.ChecksumAlgorithmCrc64nvme, - crc64nvme: getPtr("sV264W+gYBI="), - }, - } { + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + wrongChecksum, err := wrongChecksumForAlgorithm(algo) + if err != nil { + return err + } + // test for file object - _, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - ChecksumCRC32: el.crc32, - ChecksumCRC32C: el.crc32c, - ChecksumSHA1: el.sha1, - ChecksumSHA256: el.sha256, - ChecksumCRC64NVME: el.crc64nvme, - }, s3client) - if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + input := &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + } + setPutObjectChecksum(input, algo, &wrongChecksum) + _, err = putObjectWithData(int64((i+1)*100), input, s3client) + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil { return err } // test for directory object ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &dirObj, - ChecksumCRC32: el.crc32, - ChecksumCRC32C: el.crc32c, - ChecksumSHA1: el.sha1, - ChecksumSHA256: el.sha256, - ChecksumCRC64NVME: el.crc64nvme, - }) + input = &s3.PutObjectInput{ + Bucket: &bucket, + Key: &dirObj, + } + setPutObjectChecksum(input, algo, &wrongChecksum) + _, err = s3client.PutObject(ctx, input) cancel() - if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil { return err } } @@ -952,13 +856,20 @@ func PutObject_dir_object_checksums_success(s *S3Conf) error { {types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="}, {types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="}, {types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="}, + {types.ChecksumAlgorithmSha512, "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="}, + {types.ChecksumAlgorithmMd5, "1B2M2Y8AsgTpgAmY7PhCfg=="}, + {types.ChecksumAlgorithmXxhash64, "70bbN1HY6Zk="}, + {types.ChecksumAlgorithmXxhash3, "LQaABTjTlMI="}, + {types.ChecksumAlgorithmXxhash128, "maoG0wFHmNhgAcMkRo1Jfw=="}, } { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + input := &s3.PutObjectInput{ Bucket: &bucket, Key: getPtr(fmt.Sprintf("obj-%v/", i)), ChecksumAlgorithm: test.algo, - }) + } + setPutObjectChecksum(input, test.algo, &test.checksumValue) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObject(ctx, input) cancel() if err != nil { return err @@ -968,27 +879,8 @@ func PutObject_dir_object_checksums_success(s *S3Conf) error { return fmt.Errorf("expected the checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType) } - switch test.algo { - case types.ChecksumAlgorithmCrc32: - if getString(res.ChecksumCRC32) != test.checksumValue { - return fmt.Errorf("expected the crc32 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32)) - } - case types.ChecksumAlgorithmCrc32c: - if getString(res.ChecksumCRC32C) != test.checksumValue { - return fmt.Errorf("expected the crc32c checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32C)) - } - case types.ChecksumAlgorithmSha1: - if getString(res.ChecksumSHA1) != test.checksumValue { - return fmt.Errorf("expected the sha1 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA1)) - } - case types.ChecksumAlgorithmSha256: - if getString(res.ChecksumSHA256) != test.checksumValue { - return fmt.Errorf("expected the sha256 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA256)) - } - case types.ChecksumAlgorithmCrc64nvme: - if getString(res.ChecksumCRC64NVME) != test.checksumValue { - return fmt.Errorf("expected the crc64nvme checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC64NVME)) - } + if got := getString(getPutObjectChecksum(res, test.algo)); got != test.checksumValue { + return fmt.Errorf("expected the %s checksum value to be %s, instead got %s", test.algo, test.checksumValue, got) } } @@ -1006,7 +898,7 @@ func PutObject_checksums_success(s *S3Conf) error { Bucket: &bucket, Key: &obj, ChecksumAlgorithm: algo, - }, s3client) + }, s3client, withPutObjectChecksumAlgo(algo)) if err != nil { return err } @@ -1015,27 +907,8 @@ func PutObject_checksums_success(s *S3Conf) error { return fmt.Errorf("expected the object checksum type to be %v, instead got %v", types.ChecksumTypeFullObject, res.res.ChecksumType) } - switch algo { - case types.ChecksumAlgorithmCrc32: - if res.res.ChecksumCRC32 == nil { - return fmt.Errorf("expected non empty crc32 checksum in the response") - } - case types.ChecksumAlgorithmCrc32c: - if res.res.ChecksumCRC32C == nil { - return fmt.Errorf("expected non empty crc32c checksum in the response") - } - case types.ChecksumAlgorithmSha1: - if res.res.ChecksumSHA1 == nil { - return fmt.Errorf("expected non empty sha1 checksum in the response") - } - case types.ChecksumAlgorithmSha256: - if res.res.ChecksumSHA256 == nil { - return fmt.Errorf("expected non empty sha256 checksum in the response") - } - case types.ChecksumAlgorithmCrc64nvme: - if res.res.ChecksumCRC64NVME == nil { - return fmt.Errorf("expected non empty crc64nvme checksum in the response") - } + if getPutObjectChecksum(res.res, algo) == nil { + return fmt.Errorf("expected non empty %s checksum in the response", algo) } } diff --git a/tests/integration/UploadPart.go b/tests/integration/UploadPart.go index 7bf26fce..1b394b34 100644 --- a/tests/integration/UploadPart.go +++ b/tests/integration/UploadPart.go @@ -155,95 +155,26 @@ func UploadPart_invalid_checksum_header(s *S3Conf) error { partNumber := int32(1) - for _, el := range []struct { - algo string - crc32 *string - crc32c *string - sha1 *string - sha256 *string - crc64nvme *string - }{ - // CRC32 tests - { - algo: "crc32", - crc32: getPtr(""), - }, - { - algo: "crc32", - crc32: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "crc32", - crc32: getPtr("YXNrZGpoZ2tqYXNo"), // valid base64 but not crc32 - }, - // CRC32C tests - { - algo: "crc32c", - crc32c: getPtr(""), - }, - { - algo: "crc32c", - crc32c: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "crc32c", - crc32c: getPtr("c2RhZnNhZGZzZGFm"), // valid base64 but not crc32c - }, - // SHA1 tests - { - algo: "sha1", - sha1: getPtr(""), - }, - { - algo: "sha1", - sha1: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "sha1", - sha1: getPtr("c2RhZmRhc2Zkc2Fmc2RhZnNhZGZzYWRm"), // valid base64 but not sha1 - }, - // SHA256 tests - { - algo: "sha256", - sha256: getPtr(""), - }, - { - algo: "sha256", - sha256: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "sha256", - sha256: getPtr("ZGZnbmRmZ2hoZmRoZmdkaA=="), // valid base64 but not sha56 - }, - // CRC64NVME tests - { - algo: "crc64nvme", - crc64nvme: getPtr(""), - }, - { - algo: "crc64nvme", - crc64nvme: getPtr("invalid_base64!"), // invalid base64 - }, - { - algo: "crc64nvme", - crc64nvme: getPtr("ZHNhZmRzYWZzZGFmZHNhZg=="), // valid base64 but not crc64nvme - }, - } { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - ChecksumCRC32: el.crc32, - ChecksumCRC32C: el.crc32c, - ChecksumSHA1: el.sha1, - ChecksumSHA256: el.sha256, - ChecksumCRC64NVME: el.crc64nvme, - PartNumber: &partNumber, - UploadId: mp.UploadId, - }) - cancel() - if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", el.algo))); err != nil { - return err + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + // tests against: + // - empty string + // - invalid base64 + // - valid base64, but invalid checksum + for _, checksum := range []string{"", "invalid_base64!", "c2RhZnNhZGZzZGFm"} { + input := &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &partNumber, + UploadId: mp.UploadId, + } + setUploadPartChecksum(input, algo, getPtr(checksum)) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, input) + cancel() + if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(checksumHeaderName(algo))); err != nil { + return err + } } } @@ -336,36 +267,13 @@ func UploadPart_incorrect_checksums(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - for _, el := range []struct { - algo types.ChecksumAlgorithm - crc32 *string - crc32c *string - sha1 *string - sha256 *string - crc64nvme *string - }{ - { - algo: types.ChecksumAlgorithmCrc32, - crc32: getPtr("DUoRhQ=="), - }, - { - algo: types.ChecksumAlgorithmCrc32c, - crc32c: getPtr("yZRlqg=="), - }, - { - algo: types.ChecksumAlgorithmSha1, - sha1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), - }, - { - algo: types.ChecksumAlgorithmSha256, - sha256: getPtr("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek="), - }, - { - algo: types.ChecksumAlgorithmCrc64nvme, - crc64nvme: getPtr("MN2ofvMjpIQ="), - }, - } { - mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo)) + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + wrongChecksum, err := wrongChecksumForAlgorithm(algo) + if err != nil { + return err + } + + mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) if err != nil { return err } @@ -374,20 +282,17 @@ func UploadPart_incorrect_checksums(s *S3Conf) error { partNumber := int32(1) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - ChecksumCRC32: el.crc32, - ChecksumCRC32C: el.crc32c, - ChecksumSHA1: el.sha1, - ChecksumSHA256: el.sha256, - ChecksumCRC64NVME: el.crc64nvme, - UploadId: mp.UploadId, - PartNumber: &partNumber, - Body: body, - }) + input := &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + Body: body, + } + setUploadPartChecksum(input, algo, &wrongChecksum) + _, err = s3client.UploadPart(ctx, input) cancel() - if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(algo)); err != nil { return err } } @@ -479,6 +384,11 @@ func UploadPart_no_checksum_with_composite_checksum_type(s *S3Conf) error { types.ChecksumAlgorithmCrc32c, types.ChecksumAlgorithmSha1, types.ChecksumAlgorithmSha256, + types.ChecksumAlgorithmSha512, + types.ChecksumAlgorithmMd5, + types.ChecksumAlgorithmXxhash64, + types.ChecksumAlgorithmXxhash3, + types.ChecksumAlgorithmXxhash128, } { mp, err := createMp(customClient, bucket, obj, withChecksum(algo), withChecksumType(types.ChecksumTypeComposite)) if err != nil { @@ -493,8 +403,8 @@ func UploadPart_no_checksum_with_composite_checksum_type(s *S3Conf) error { }) } -func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Conf) error { - testName := "UploadPart_should_calculate_checksum_if_only_algorithm_is_provided" +func UploadPart_with_checksums_success(s *S3Conf) error { + testName := "UploadPart_with_checksums_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { customClient := s3.NewFromConfig(s.Config(), func(o *s3.Options) { o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset @@ -512,6 +422,11 @@ func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Con {types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c}, {types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1}, {types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmSha512}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmMd5}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash64}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash3}, + {types.ChecksumTypeComposite, types.ChecksumAlgorithmXxhash128}, } { mp, err := createMp(customClient, bucket, obj, withChecksum(test.chAlgo), withChecksumType(test.chType)) if err != nil { @@ -527,91 +442,14 @@ func UploadPart_should_calculate_checksum_if_only_algorithm_is_provided(s *S3Con return fmt.Errorf("expected 1 uploaded part, instaed got %d", len(parts)) } - part := parts[0] - switch test.chAlgo { - case types.ChecksumAlgorithmCrc32: - if getString(part.ChecksumCRC32) != csum { - return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32)) - } - case types.ChecksumAlgorithmCrc32c: - if getString(part.ChecksumCRC32C) != csum { - return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC32C)) - } - case types.ChecksumAlgorithmCrc64nvme: - if getString(part.ChecksumCRC64NVME) != csum { - return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumCRC64NVME)) - } - case types.ChecksumAlgorithmSha1: - if getString(part.ChecksumSHA1) != csum { - return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA1)) - } - case types.ChecksumAlgorithmSha256: - if getString(part.ChecksumSHA256) != csum { - return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, getString(part.ChecksumSHA256)) - } + if got := getString(getPartChecksum(parts[0], test.chAlgo)); got != csum { + return fmt.Errorf("expected the uploaded part checksum %s to be %s, instead got %s", test.chAlgo, csum, got) } } return nil }) } -func UploadPart_with_checksums_success(s *S3Conf) error { - testName := "UploadPart_with_checksums_success" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - - for i, algo := range types.ChecksumAlgorithmCrc32.Values() { - mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) - if err != nil { - return err - } - - partNumber := int32(1) - data := make([]byte, i*100) - rand.Read(data) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - ChecksumAlgorithm: algo, - UploadId: mp.UploadId, - PartNumber: &partNumber, - Body: bytes.NewReader(data), - }) - cancel() - if err != nil { - return err - } - - switch algo { - case types.ChecksumAlgorithmCrc32: - if res.ChecksumCRC32 == nil { - return fmt.Errorf("expected non empty crc32 checksum in the response") - } - case types.ChecksumAlgorithmCrc32c: - if res.ChecksumCRC32C == nil { - return fmt.Errorf("expected non empty crc32c checksum in the response") - } - case types.ChecksumAlgorithmSha1: - if res.ChecksumSHA1 == nil { - return fmt.Errorf("expected non empty sha1 checksum in the response") - } - case types.ChecksumAlgorithmSha256: - if res.ChecksumSHA256 == nil { - return fmt.Errorf("expected non empty sha256 checksum in the response") - } - case types.ChecksumAlgorithmCrc64nvme: - if res.ChecksumCRC64NVME == nil { - return fmt.Errorf("expected non empty crc64nvme checksum in the response") - } - } - } - - return nil - }) -} - func UploadPart_non_existing_key(s *S3Conf) error { testName := "UploadPart_non_existing_key" partNumber := int32(1) diff --git a/tests/integration/UploadPartCopy.go b/tests/integration/UploadPartCopy.go index 8201323b..f5f86c4e 100644 --- a/tests/integration/UploadPartCopy.go +++ b/tests/integration/UploadPartCopy.go @@ -845,57 +845,50 @@ func UploadPartCopy_conditional_reads(s *S3Conf) error { func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error { testName := "UploadPartCopy_should_copy_the_checksum" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - srcObj := "source-object" + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + obj := fmt.Sprintf("my-obj-%v", i+1) + srcObj := fmt.Sprintf("source-object-%v", i+1) - mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) - if err != nil { - return err - } + mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) + if err != nil { + return err + } - out, err := putObjectWithData(300, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &srcObj, - ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, - }, s3client) - if err != nil { - return err - } + out, err := putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: algo, + }, s3client, withPutObjectChecksumAlgo(algo)) + if err != nil { + return err + } - partNumber := int32(1) + partNumber := int32(1) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mp.UploadId, - PartNumber: &partNumber, - CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } - if getString(res.CopyPartResult.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { - return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", - getString(out.res.ChecksumCRC32), getString(res.CopyPartResult.ChecksumCRC32)) - } - if res.CopyPartResult.ChecksumCRC32C != nil { - return fmt.Errorf("expected nil crc32c checksum, instead got %v", - *res.CopyPartResult.ChecksumCRC32C) - } - if res.CopyPartResult.ChecksumSHA1 != nil { - return fmt.Errorf("expected nil sha1 checksum, instead got %v", - *res.CopyPartResult.ChecksumSHA1) - } - if res.CopyPartResult.ChecksumSHA256 != nil { - return fmt.Errorf("expected nil sha256 checksum, instead got %v", - *res.CopyPartResult.ChecksumSHA256) - } - if res.CopyPartResult.ChecksumCRC64NVME != nil { - return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", - *res.CopyPartResult.ChecksumCRC64NVME) + if got, expected := getString(getUploadPartCopyChecksum(res.CopyPartResult, algo)), getString(getPutObjectChecksum(out.res, algo)); got != expected { + return fmt.Errorf("expected %s checksum to be %v, instead got %v", algo, expected, got) + } + for _, otherAlgo := range types.ChecksumAlgorithmCrc32.Values() { + if otherAlgo == algo { + continue + } + if checksum := getUploadPartCopyChecksum(res.CopyPartResult, otherAlgo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", otherAlgo, *checksum) + } + } } return nil @@ -937,25 +930,10 @@ func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error { return err } - if res.CopyPartResult.ChecksumCRC32 != nil { - return fmt.Errorf("expected nil crc32 checksum, instead got %v", - *res.CopyPartResult.ChecksumCRC32) - } - if res.CopyPartResult.ChecksumCRC32C != nil { - return fmt.Errorf("expected nil crc32c checksum, instead got %v", - *res.CopyPartResult.ChecksumCRC32C) - } - if res.CopyPartResult.ChecksumSHA1 != nil { - return fmt.Errorf("expected nil sha1 checksum, instead got %v", - *res.CopyPartResult.ChecksumSHA1) - } - if res.CopyPartResult.ChecksumSHA256 != nil { - return fmt.Errorf("expected nil sha256 checksum, instead got %v", - *res.CopyPartResult.ChecksumSHA256) - } - if res.CopyPartResult.ChecksumCRC64NVME != nil { - return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", - *res.CopyPartResult.ChecksumCRC64NVME) + for _, algo := range types.ChecksumAlgorithmCrc32.Values() { + if checksum := getUploadPartCopyChecksum(res.CopyPartResult, algo); checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum) + } } return nil @@ -965,56 +943,51 @@ func UploadPartCopy_should_not_copy_the_checksum(s *S3Conf) error { func UploadPartCopy_should_calculate_the_checksum(s *S3Conf) error { testName := "UploadPartCopy_should_calculate_the_checksum" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - srcObj := "source-object" + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + obj := fmt.Sprintf("my-obj-%v", i+1) + srcObj := fmt.Sprintf("source-object-%v", i+1) - mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha256)) - if err != nil { - return err - } + mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) + if err != nil { + return err + } - _, err = putObjectWithData(300, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &srcObj, - ChecksumAlgorithm: types.ChecksumAlgorithmSha1, // different from the mp checksum (sha256) - }, s3client) - if err != nil { - return err - } + _, err = putObjectWithData(300, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ChecksumAlgorithm: types.ChecksumAlgorithmSha1, + }, s3client) + if err != nil { + return err + } - partNumber := int32(1) + partNumber := int32(1) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mp.UploadId, - PartNumber: &partNumber, - CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), - }) - cancel() - if err != nil { - return err - } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + PartNumber: &partNumber, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } - if res.CopyPartResult.ChecksumCRC32 != nil { - return fmt.Errorf("expected nil crc32 checksum, instead got %v", - *res.CopyPartResult.ChecksumCRC32) - } - if res.CopyPartResult.ChecksumCRC32C != nil { - return fmt.Errorf("expected nil crc32c checksum, instead got %v", - *res.CopyPartResult.ChecksumCRC32C) - } - if res.CopyPartResult.ChecksumCRC64NVME != nil { - return fmt.Errorf("expected nil crc64nvme checksum, instead got %v", - *res.CopyPartResult.ChecksumCRC64NVME) - } - if res.CopyPartResult.ChecksumSHA1 != nil { - return fmt.Errorf("expected nil sha1 checksum, instead got %v", - *res.CopyPartResult.ChecksumSHA1) - } - if getString(res.CopyPartResult.ChecksumSHA256) == "" { - return fmt.Errorf("expected non empty sha256 checksum") + for _, otherAlgo := range types.ChecksumAlgorithmCrc32.Values() { + checksum := getUploadPartCopyChecksum(res.CopyPartResult, otherAlgo) + if otherAlgo == algo { + if getString(checksum) == "" { + return fmt.Errorf("expected non empty %s checksum", algo) + } + continue + } + if checksum != nil { + return fmt.Errorf("expected nil %s checksum, instead got %v", otherAlgo, *checksum) + } + } } return nil diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 0b324677..fb1f4aef 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -445,7 +445,6 @@ func TestUploadPart(ts *TestState) { ts.Run(UploadPart_incorrect_checksums) ts.Run(UploadPart_no_checksum_with_full_object_checksum_type) ts.Run(UploadPart_no_checksum_with_composite_checksum_type) - ts.Run(UploadPart_should_calculate_checksum_if_only_algorithm_is_provided) ts.Run(UploadPart_with_checksums_success) } ts.Run(UploadPart_success) @@ -1589,7 +1588,6 @@ func GetIntTests() IntTests { "UploadPart_incorrect_checksums": UploadPart_incorrect_checksums, "UploadPart_no_checksum_with_full_object_checksum_type": UploadPart_no_checksum_with_full_object_checksum_type, "UploadPart_no_checksum_with_composite_checksum_type": UploadPart_no_checksum_with_composite_checksum_type, - "UploadPart_should_calculate_checksum_if_only_algorithm_is_provided": UploadPart_should_calculate_checksum_if_only_algorithm_is_provided, "UploadPart_with_checksums_success": UploadPart_with_checksums_success, "UploadPart_success": UploadPart_success, "UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket, diff --git a/tests/integration/signed_streaming_payload_trailer.go b/tests/integration/signed_streaming_payload_trailer.go index 5a16aa83..8e2ed85a 100644 --- a/tests/integration/signed_streaming_payload_trailer.go +++ b/tests/integration/signed_streaming_payload_trailer.go @@ -108,24 +108,17 @@ func SignedStreamingPayloadTrailer_invalid_checksum(s *S3Conf) error { testName := "SignedStreamingPayloadTrailer_invalid_checksum" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { object := "my-object" - for i, test := range []struct { - trailerHdr string - trailingChecksum string - }{ - {"x-amz-checksum-crc32", "x-amz-checksum-crc32:invalid"}, - {"x-amz-checksum-crc32c", "x-amz-checksum-crc32c:invalid"}, - {"x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:invalid"}, - {"x-amz-checksum-sha1", "x-amz-checksum-sha1:invalid"}, - {"x-amz-checksum-sha256", "x-amz-checksum-sha256:invalid"}, - } { - _, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{ - "x-amz-trailer": test.trailerHdr, + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + trailerHdr := checksumHeaderName(algo) + trailingChecksum := fmt.Sprintf("%s:invalid", trailerHdr) + _, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("dummy data"), withTrailingChecksum(trailingChecksum), withCustomHeaders(map[string]string{ + "x-amz-trailer": trailerHdr, })) if err != nil { return fmt.Errorf("test %v failed: %w", i+1, err) } - if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(test.trailerHdr), apiErr); err != nil { + if err := compareS3ApiError(s3err.GetInvalidTrailingChecksumHeaderErr(trailerHdr), apiErr); err != nil { return fmt.Errorf("test %v failed: %w", i+1, err) } } @@ -139,18 +132,24 @@ func SignedStreamingPayloadTrailer_bad_digest(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { object := "my-object" for i, test := range []struct { - algo types.ChecksumAlgorithm - trailerHdr string - trailingChecksum string + algo types.ChecksumAlgorithm + checksum string }{ - {types.ChecksumAlgorithmCrc32, "x-amz-checksum-crc32", "x-amz-checksum-crc32:NhCmhg=="}, - {types.ChecksumAlgorithmCrc32c, "x-amz-checksum-crc32c", "x-amz-checksum-crc32c:+Cy97w=="}, - {types.ChecksumAlgorithmCrc64nvme, "x-amz-checksum-crc64nvme", "x-amz-checksum-crc64nvme:QFRKMGE3tuw="}, - {types.ChecksumAlgorithmSha1, "x-amz-checksum-sha1", "x-amz-checksum-sha1:qvTGHdzF6KLavt4PO0gs2a6pQ00="}, - {types.ChecksumAlgorithmSha256, "x-amz-checksum-sha256", "x-amz-checksum-sha256:LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="}, + {types.ChecksumAlgorithmCrc32, "NhCmhg=="}, + {types.ChecksumAlgorithmCrc32c, "+Cy97w=="}, + {types.ChecksumAlgorithmCrc64nvme, "QFRKMGE3tuw="}, + {types.ChecksumAlgorithmSha1, "qvTGHdzF6KLavt4PO0gs2a6pQ00="}, + {types.ChecksumAlgorithmSha256, "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="}, + {types.ChecksumAlgorithmSha512, "mD1D3f9tqQ9qXTthckRqH/4ii4A/5k/dXc+rVkYHioloUf6C9iPJ1uVlSz0vNjoE7BfPtitgdDepx8Ey1RHlIg=="}, + {types.ChecksumAlgorithmMd5, "Q3uTDbhLgHnC3YBKcZNrXw=="}, + {types.ChecksumAlgorithmXxhash64, "dVz0EuroMyA="}, + {types.ChecksumAlgorithmXxhash3, "OHmyyhTeg1E="}, + {types.ChecksumAlgorithmXxhash128, "6fhA2KZsN830zx0e5P183w=="}, } { - _, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(test.trailingChecksum), withCustomHeaders(map[string]string{ - "x-amz-trailer": test.trailerHdr, + trailerHdr := checksumHeaderName(test.algo) + trailingChecksum := fmt.Sprintf("%s:%s", trailerHdr, test.checksum) + _, apiErr, err := testSignedStreamingObjectPut(s, bucket, object, []byte("some random data"), withTrailingChecksum(trailingChecksum), withCustomHeaders(map[string]string{ + "x-amz-trailer": trailerHdr, })) if err != nil { return fmt.Errorf("test %v failed: %w", i+1, err) @@ -178,6 +177,11 @@ func SignedStreamingPayloadTrailer_success(s *S3Conf) error { {"x-amz-checksum-crc64nvme", "dYnI3/Fh0gM="}, {"x-amz-checksum-sha1", "8O8FwCfmd5fCbCBvH09mrKMVoHU="}, {"x-amz-checksum-sha256", "OoSow5X4zTIPl27MtdFdYT+9O3C367C75+Cb2MFtRBc="}, + {"x-amz-checksum-sha512", "uiz+VuENyLjzFxWrdmbN/NdIdldj/V3saJF6FsckcYu6xu26fM0CfAaTySaoFJmIdI2m5wbYMJxtShQ1PXk3Tg=="}, + {"x-amz-checksum-md5", "EtWtthXtCk3RxUiXKw+ydw=="}, + {"x-amz-checksum-xxhash64", "McmxUfNFLUs="}, + {"x-amz-checksum-xxhash3", "tLh+aNltTM4="}, + {"x-amz-checksum-xxhash128", "MXqrLVobrQkTBf82+7i9AQ=="}, } { headers, apiErr, err := testSignedStreamingObjectPut( s, diff --git a/tests/integration/unsigned_streaming_payload_trailer.go b/tests/integration/unsigned_streaming_payload_trailer.go index da92fda1..505c30f2 100644 --- a/tests/integration/unsigned_streaming_payload_trailer.go +++ b/tests/integration/unsigned_streaming_payload_trailer.go @@ -134,6 +134,11 @@ func UnsignedStreamingPayloadTrailer_multiple_checksum_headers(s *S3Conf) error {"crc64nvme", "QFRKMGE3tuw="}, {"sha1", "qvTGHdzF6KLavt4PO0gs2a6pQ00="}, {"sha256", "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="}, + {"sha512", "uiz+VuENyLjzFxWrdmbN/NdIdldj/V3saJF6FsckcYu6xu26fM0CfAaTySaoFJmIdI2m5wbYMJxtShQ1PXk3Tg=="}, + {"md5", "EtWtthXtCk3RxUiXKw+ydw=="}, + {"xxhash64", "McmxUfNFLUs="}, + {"xxhash3", "tLh+aNltTM4="}, + {"xxhash128", "MXqrLVobrQkTBf82+7i9AQ=="}, } { reqHeaders := map[string]string{ "x-amz-decoded-content-length": "5", @@ -348,6 +353,11 @@ func UnsignedStreamingPayloadTrailer_no_payload_trailer_only_headers(s *S3Conf) {"crc64nvme", "SmzZ/LTp1CA="}, {"sha1", "L7XhNBn8iSRoZeejJPR27GJOh0A="}, {"sha256", "fRpUEnsiJQL1t5tfsIAwYRUqRPkrN+I8ZSe69mXU2po="}, + {"sha512", "1xakGIVptoqxtt+sF45XARTN8Oo6HMDjFIbD5BJBvGp2Qk6MN6sm8Jb8he+YhsjLY0GH9P3f9kX7CZ8f9UxrjA=="}, + {"md5", "esZsDxSN6VGbi9JkMSxNZA=="}, + {"xxhash64", "GGCUDikCgi0="}, + {"xxhash3", "WkDcP9RMBS8="}, + {"xxhash128", "Kq/YOGmlnDE/55jA7aptxg=="}, } { csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key) reqHeaders := map[string]string{ @@ -387,6 +397,11 @@ func UnsignedStreamingPayloadTrailer_success_both_sdk_algo_and_trailer(s *S3Conf {"crc64nvme", "SmzZ/LTp1CA="}, {"sha1", "L7XhNBn8iSRoZeejJPR27GJOh0A="}, {"sha256", "fRpUEnsiJQL1t5tfsIAwYRUqRPkrN+I8ZSe69mXU2po="}, + {"sha512", "1xakGIVptoqxtt+sF45XARTN8Oo6HMDjFIbD5BJBvGp2Qk6MN6sm8Jb8he+YhsjLY0GH9P3f9kX7CZ8f9UxrjA=="}, + {"md5", "esZsDxSN6VGbi9JkMSxNZA=="}, + {"xxhash64", "GGCUDikCgi0="}, + {"xxhash3", "WkDcP9RMBS8="}, + {"xxhash128", "Kq/YOGmlnDE/55jA7aptxg=="}, } { csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key) reqHeaders := map[string]string{ @@ -508,23 +523,29 @@ func UnsignedStreamingPayloadTrailer_UploadPart_success_with_trailer(s *S3Conf) object := "my-object" for i, test := range []struct { - key string + algo types.ChecksumAlgorithm value string }{ - {"crc32", "QWaN2w=="}, - {"crc32c", "R/I7iQ=="}, - {"crc64nvme", "dPVWc2vU1+Q="}, - {"sha1", "YR/1TvTYOJz5gtqVFoBJBtmTibY="}, - {"sha256", "eXuwq/95jXIAr3aF3KeQHt/8Ur8mUA1b2XKCZY7iQVI="}, + {types.ChecksumAlgorithmCrc32, "QWaN2w=="}, + {types.ChecksumAlgorithmCrc32c, "R/I7iQ=="}, + {types.ChecksumAlgorithmCrc64nvme, "dPVWc2vU1+Q="}, + {types.ChecksumAlgorithmSha1, "YR/1TvTYOJz5gtqVFoBJBtmTibY="}, + {types.ChecksumAlgorithmSha256, "eXuwq/95jXIAr3aF3KeQHt/8Ur8mUA1b2XKCZY7iQVI="}, + {types.ChecksumAlgorithmSha512, "6tQCd6X50h2wXt5deAY0eKF5Xb1LLbSvkZt+Bqczz8bd1+rH+VYZSgWhjOG4zJ41K0kLmDQxuDyOeHMKqKQGNA=="}, + {types.ChecksumAlgorithmMd5, "Mb+5cwrlGvc5U7pyDIZg1w=="}, + {types.ChecksumAlgorithmXxhash64, "vYOmFpsGXtY="}, + {types.ChecksumAlgorithmXxhash3, "9J6xx21X+f8="}, + {types.ChecksumAlgorithmXxhash128, "menoCZOCZ2Acv4XMxaUR9w=="}, } { - mp, err := createMp(s3client, bucket, object, withChecksum(types.ChecksumAlgorithm(strings.ToUpper(test.key)))) + + mp, err := createMp(s3client, bucket, object, withChecksum(test.algo)) if err != nil { return err } - csumHdr := fmt.Sprintf("x-amz-checksum-%s", test.key) + csumHdr := checksumHeaderName(test.algo) reqHeaders := map[string]string{ "x-amz-decoded-content-length": "10", - "x-amz-sdk-checksum-algorithm": test.key, + "x-amz-sdk-checksum-algorithm": strings.ToLower(string(test.algo)), "x-amz-trailer": csumHdr, } body := bytes.NewBuffer([]byte("A\r\ndummy data\r\n0\r\n")) diff --git a/tests/integration/utils.go b/tests/integration/utils.go index f30d847b..d1489702 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -22,6 +22,7 @@ import ( "crypto/rand" "crypto/sha1" "crypto/sha256" + "crypto/sha512" "encoding/base64" "encoding/hex" "encoding/json" @@ -53,7 +54,9 @@ import ( "github.com/aws/smithy-go" "github.com/aws/smithy-go/middleware" smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/cespare/xxhash/v2" "github.com/versity/versitygw/s3err" + "github.com/zeebo/xxh3" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) @@ -547,21 +550,46 @@ func hasPrefixName(prefixes []types.CommonPrefix, names []string) bool { return true } +type putObjectCfg struct { + checksumAlgorithm types.ChecksumAlgorithm +} + +type putObjectOpt func(*putObjectCfg) + +func withPutObjectChecksumAlgo(algo types.ChecksumAlgorithm) putObjectOpt { + return func(poc *putObjectCfg) { poc.checksumAlgorithm = algo } +} + type putObjectOutput struct { csum [32]byte data []byte res *s3.PutObjectOutput } -func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) { +func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client, opts ...putObjectOpt) (*putObjectOutput, error) { + cfg := &putObjectCfg{} + for _, opt := range opts { + opt(cfg) + } + var csum [32]byte var data []byte if input.Body == nil && lgth != 0 { data = make([]byte, lgth) rand.Read(data) + csum = sha256.Sum256(data) - r := bytes.NewReader(data) - input.Body = r + if cfg.checksumAlgorithm != "" { + hasher, err := NewHasher(cfg.checksumAlgorithm) + if err != nil { + return nil, err + } + + hasher.Write(data) + sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + setPutObjectChecksum(input, cfg.checksumAlgorithm, &sum) + } + input.Body = bytes.NewReader(data) } ctx, cancel := context.WithTimeout(context.Background(), longTimeout) @@ -713,6 +741,36 @@ func compareParts(parts1, parts2 []types.Part) bool { return false } } + if prt.ChecksumSHA512 != nil { + if *prt.ChecksumSHA512 != getString(parts2[i].ChecksumSHA512) { + fmt.Printf("sha512 checksums are not equal, %v != %v\n", *prt.ChecksumSHA512, getString(parts2[i].ChecksumSHA512)) + return false + } + } + if prt.ChecksumMD5 != nil { + if *prt.ChecksumMD5 != getString(parts2[i].ChecksumMD5) { + fmt.Printf("md5 checksums are not equal, %v != %v\n", *prt.ChecksumMD5, getString(parts2[i].ChecksumMD5)) + return false + } + } + if prt.ChecksumXXHASH64 != nil { + if *prt.ChecksumXXHASH64 != getString(parts2[i].ChecksumXXHASH64) { + fmt.Printf("xxhash64 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH64, getString(parts2[i].ChecksumXXHASH64)) + return false + } + } + if prt.ChecksumXXHASH3 != nil { + if *prt.ChecksumXXHASH3 != getString(parts2[i].ChecksumXXHASH3) { + fmt.Printf("xxhash3 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH3, getString(parts2[i].ChecksumXXHASH3)) + return false + } + } + if prt.ChecksumXXHASH128 != nil { + if *prt.ChecksumXXHASH128 != getString(parts2[i].ChecksumXXHASH128) { + fmt.Printf("xxhash128 checksums are not equal, %v != %v\n", *prt.ChecksumXXHASH128, getString(parts2[i].ChecksumXXHASH128)) + return false + } + } } return true } @@ -775,6 +833,299 @@ func getPtr[T any](str T) *T { return &str } +func checksumHeaderName(algo types.ChecksumAlgorithm) string { + return fmt.Sprintf("x-amz-checksum-%s", strings.ToLower(string(algo))) +} + +type checksumFields struct { + CRC32 **string + CRC32C **string + SHA1 **string + SHA256 **string + CRC64NVME **string + SHA512 **string + MD5 **string + XXHASH64 **string + XXHASH3 **string + XXHASH128 **string +} + +func selectChecksum(algo types.ChecksumAlgorithm, fields checksumFields) **string { + switch algo { + case types.ChecksumAlgorithmCrc32: + return fields.CRC32 + case types.ChecksumAlgorithmCrc32c: + return fields.CRC32C + case types.ChecksumAlgorithmSha1: + return fields.SHA1 + case types.ChecksumAlgorithmSha256: + return fields.SHA256 + case types.ChecksumAlgorithmCrc64nvme: + return fields.CRC64NVME + case types.ChecksumAlgorithmSha512: + return fields.SHA512 + case types.ChecksumAlgorithmMd5: + return fields.MD5 + case types.ChecksumAlgorithmXxhash64: + return fields.XXHASH64 + case types.ChecksumAlgorithmXxhash3: + return fields.XXHASH3 + case types.ChecksumAlgorithmXxhash128: + return fields.XXHASH128 + default: + return nil + } +} + +func getChecksum(algo types.ChecksumAlgorithm, fields checksumFields) *string { + if checksum := selectChecksum(algo, fields); checksum != nil { + return *checksum + } + return nil +} + +func setChecksum(algo types.ChecksumAlgorithm, fields checksumFields, checksum *string) { + if selected := selectChecksum(algo, fields); selected != nil { + *selected = checksum + } +} + +func getPartChecksum(part types.Part, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &part.ChecksumCRC32, + CRC32C: &part.ChecksumCRC32C, + SHA1: &part.ChecksumSHA1, + SHA256: &part.ChecksumSHA256, + CRC64NVME: &part.ChecksumCRC64NVME, + SHA512: &part.ChecksumSHA512, + MD5: &part.ChecksumMD5, + XXHASH64: &part.ChecksumXXHASH64, + XXHASH3: &part.ChecksumXXHASH3, + XXHASH128: &part.ChecksumXXHASH128, + }) +} + +func setPartChecksum(part *types.Part, algo types.ChecksumAlgorithm, checksum *string) { + setChecksum(algo, checksumFields{ + CRC32: &part.ChecksumCRC32, + CRC32C: &part.ChecksumCRC32C, + SHA1: &part.ChecksumSHA1, + SHA256: &part.ChecksumSHA256, + CRC64NVME: &part.ChecksumCRC64NVME, + SHA512: &part.ChecksumSHA512, + MD5: &part.ChecksumMD5, + XXHASH64: &part.ChecksumXXHASH64, + XXHASH3: &part.ChecksumXXHASH3, + XXHASH128: &part.ChecksumXXHASH128, + }, checksum) +} + +func getCompletedPartChecksum(part types.CompletedPart, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &part.ChecksumCRC32, + CRC32C: &part.ChecksumCRC32C, + SHA1: &part.ChecksumSHA1, + SHA256: &part.ChecksumSHA256, + CRC64NVME: &part.ChecksumCRC64NVME, + SHA512: &part.ChecksumSHA512, + MD5: &part.ChecksumMD5, + XXHASH64: &part.ChecksumXXHASH64, + XXHASH3: &part.ChecksumXXHASH3, + XXHASH128: &part.ChecksumXXHASH128, + }) +} + +func completedPartFromPart(part types.Part) types.CompletedPart { + return types.CompletedPart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + ChecksumCRC32: part.ChecksumCRC32, + ChecksumCRC32C: part.ChecksumCRC32C, + ChecksumSHA1: part.ChecksumSHA1, + ChecksumSHA256: part.ChecksumSHA256, + ChecksumCRC64NVME: part.ChecksumCRC64NVME, + ChecksumSHA512: part.ChecksumSHA512, + ChecksumMD5: part.ChecksumMD5, + ChecksumXXHASH64: part.ChecksumXXHASH64, + ChecksumXXHASH3: part.ChecksumXXHASH3, + ChecksumXXHASH128: part.ChecksumXXHASH128, + } +} + +func getPutObjectChecksum(out *s3.PutObjectOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func setPutObjectChecksum(in *s3.PutObjectInput, algo types.ChecksumAlgorithm, checksum *string) { + setChecksum(algo, checksumFields{ + CRC32: &in.ChecksumCRC32, + CRC32C: &in.ChecksumCRC32C, + SHA1: &in.ChecksumSHA1, + SHA256: &in.ChecksumSHA256, + CRC64NVME: &in.ChecksumCRC64NVME, + SHA512: &in.ChecksumSHA512, + MD5: &in.ChecksumMD5, + XXHASH64: &in.ChecksumXXHASH64, + XXHASH3: &in.ChecksumXXHASH3, + XXHASH128: &in.ChecksumXXHASH128, + }, checksum) +} + +func getGetObjectChecksum(out *s3.GetObjectOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func getHeadObjectChecksum(out *s3.HeadObjectOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func getObjectAttributesChecksum(out *types.Checksum, algo types.ChecksumAlgorithm) *string { + if out == nil { + return nil + } + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func getUploadPartChecksum(out *s3.UploadPartOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func setUploadPartChecksum(in *s3.UploadPartInput, algo types.ChecksumAlgorithm, checksum *string) { + setChecksum(algo, checksumFields{ + CRC32: &in.ChecksumCRC32, + CRC32C: &in.ChecksumCRC32C, + SHA1: &in.ChecksumSHA1, + SHA256: &in.ChecksumSHA256, + CRC64NVME: &in.ChecksumCRC64NVME, + SHA512: &in.ChecksumSHA512, + MD5: &in.ChecksumMD5, + XXHASH64: &in.ChecksumXXHASH64, + XXHASH3: &in.ChecksumXXHASH3, + XXHASH128: &in.ChecksumXXHASH128, + }, checksum) +} + +func getCompleteMultipartUploadChecksum(out *s3.CompleteMultipartUploadOutput, algo types.ChecksumAlgorithm) *string { + return getChecksum(algo, checksumFields{ + CRC32: &out.ChecksumCRC32, + CRC32C: &out.ChecksumCRC32C, + SHA1: &out.ChecksumSHA1, + SHA256: &out.ChecksumSHA256, + CRC64NVME: &out.ChecksumCRC64NVME, + SHA512: &out.ChecksumSHA512, + MD5: &out.ChecksumMD5, + XXHASH64: &out.ChecksumXXHASH64, + XXHASH3: &out.ChecksumXXHASH3, + XXHASH128: &out.ChecksumXXHASH128, + }) +} + +func setCompleteMultipartUploadChecksum(in *s3.CompleteMultipartUploadInput, algo types.ChecksumAlgorithm, checksum *string) { + setChecksum(algo, checksumFields{ + CRC32: &in.ChecksumCRC32, + CRC32C: &in.ChecksumCRC32C, + SHA1: &in.ChecksumSHA1, + SHA256: &in.ChecksumSHA256, + CRC64NVME: &in.ChecksumCRC64NVME, + SHA512: &in.ChecksumSHA512, + MD5: &in.ChecksumMD5, + XXHASH64: &in.ChecksumXXHASH64, + XXHASH3: &in.ChecksumXXHASH3, + XXHASH128: &in.ChecksumXXHASH128, + }, checksum) +} + +func getCopyObjectChecksum(result *types.CopyObjectResult, algo types.ChecksumAlgorithm) *string { + if result == nil { + return nil + } + return getChecksum(algo, checksumFields{ + CRC32: &result.ChecksumCRC32, + CRC32C: &result.ChecksumCRC32C, + SHA1: &result.ChecksumSHA1, + SHA256: &result.ChecksumSHA256, + CRC64NVME: &result.ChecksumCRC64NVME, + SHA512: &result.ChecksumSHA512, + MD5: &result.ChecksumMD5, + XXHASH64: &result.ChecksumXXHASH64, + XXHASH3: &result.ChecksumXXHASH3, + XXHASH128: &result.ChecksumXXHASH128, + }) +} + +func getUploadPartCopyChecksum(result *types.CopyPartResult, algo types.ChecksumAlgorithm) *string { + if result == nil { + return nil + } + return getChecksum(algo, checksumFields{ + CRC32: &result.ChecksumCRC32, + CRC32C: &result.ChecksumCRC32C, + SHA1: &result.ChecksumSHA1, + SHA256: &result.ChecksumSHA256, + CRC64NVME: &result.ChecksumCRC64NVME, + SHA512: &result.ChecksumSHA512, + MD5: &result.ChecksumMD5, + XXHASH64: &result.ChecksumXXHASH64, + XXHASH3: &result.ChecksumXXHASH3, + XXHASH128: &result.ChecksumXXHASH128, + }) +} + // mp1 needs to be the response from the server // mp2 needs to be the expected values // The keys from the server are always converted to lowercase @@ -921,7 +1272,8 @@ func compareDelObjects(list1, list2 []types.DeletedObject) bool { func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) { partSize := size / partCount - var hash hash.Hash + var objHasher hash.Hash + var partHasher hash.Hash cfg := new(mpCfg) for _, opt := range opts { @@ -930,17 +1282,34 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId switch cfg.checksumAlgorithm { case types.ChecksumAlgorithmCrc32: - hash = crc32.NewIEEE() + objHasher = crc32.NewIEEE() case types.ChecksumAlgorithmCrc32c: - hash = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + objHasher = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case types.ChecksumAlgorithmMd5: + objHasher = md5.New() case types.ChecksumAlgorithmSha1: - hash = sha1.New() + objHasher = sha1.New() case types.ChecksumAlgorithmSha256: - hash = sha256.New() + objHasher = sha256.New() + case types.ChecksumAlgorithmSha512: + objHasher = sha512.New() case types.ChecksumAlgorithmCrc64nvme: - hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))) + objHasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))) + case types.ChecksumAlgorithmXxhash64: + objHasher = xxhash.New() + case types.ChecksumAlgorithmXxhash3: + objHasher = xxh3.New() + case types.ChecksumAlgorithmXxhash128: + objHasher = xxh3.New128() default: - hash = sha256.New() + objHasher = sha256.New() + } + + if cfg.checksumAlgorithm != "" { + partHasher, err = NewHasher(cfg.checksumAlgorithm) + if err != nil { + return nil, "", err + } } for partNumber := int64(1); partNumber <= partCount; partNumber++ { @@ -952,18 +1321,28 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId partBuffer := make([]byte, partEnd-partStart+1) rand.Read(partBuffer) - hash.Write(partBuffer) + objHasher.Write(partBuffer) + if partHasher != nil { + partHasher.Write(partBuffer) + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) pn := int32(partNumber) - out, err := client.UploadPart(ctx, &s3.UploadPartInput{ + input := &s3.UploadPartInput{ Bucket: &bucket, Key: &key, UploadId: &uploadId, Body: bytes.NewReader(partBuffer), PartNumber: &pn, ChecksumAlgorithm: cfg.checksumAlgorithm, - }) + } + if partHasher != nil { + partChecksum := base64.StdEncoding.EncodeToString(partHasher.Sum(nil)) + setUploadPartChecksum(input, cfg.checksumAlgorithm, &partChecksum) + partHasher.Reset() + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.UploadPart(ctx, input) cancel() if err != nil { return parts, "", err @@ -975,22 +1354,11 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId Size: &partSize, } - switch cfg.checksumAlgorithm { - case types.ChecksumAlgorithmCrc32: - part.ChecksumCRC32 = out.ChecksumCRC32 - case types.ChecksumAlgorithmCrc32c: - part.ChecksumCRC32C = out.ChecksumCRC32C - case types.ChecksumAlgorithmSha1: - part.ChecksumSHA1 = out.ChecksumSHA1 - case types.ChecksumAlgorithmSha256: - part.ChecksumSHA256 = out.ChecksumSHA256 - case types.ChecksumAlgorithmCrc64nvme: - part.ChecksumCRC64NVME = out.ChecksumCRC64NVME - } + setPartChecksum(&part, cfg.checksumAlgorithm, getUploadPartChecksum(out, cfg.checksumAlgorithm)) parts = append(parts, part) } - sum := hash.Sum(nil) + sum := objHasher.Sum(nil) if cfg.checksumAlgorithm == "" { csum = hex.EncodeToString(sum[:]) @@ -1277,6 +1645,26 @@ func createObjVersions(client *s3.Client, bucket, object string, count int, opts version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ types.ChecksumAlgorithmSha256, } + case r.res.ChecksumSHA512 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmSha512, + } + case r.res.ChecksumMD5 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmMd5, + } + case r.res.ChecksumXXHASH64 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmXxhash64, + } + case r.res.ChecksumXXHASH3 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmXxhash3, + } + case r.res.ChecksumXXHASH128 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmXxhash128, + } } versions = append(versions, version) @@ -2012,14 +2400,26 @@ func lockObject(client *s3.Client, mode objectLockMode, bucket, object, versionI func NewHasher(algo types.ChecksumAlgorithm) (hash.Hash, error) { var hasher hash.Hash switch algo { + case types.ChecksumAlgorithmMd5: + hasher = md5.New() case types.ChecksumAlgorithmSha256: hasher = sha256.New() + case types.ChecksumAlgorithmSha512: + hasher = sha512.New() case types.ChecksumAlgorithmSha1: hasher = sha1.New() case types.ChecksumAlgorithmCrc32: hasher = crc32.NewIEEE() case types.ChecksumAlgorithmCrc32c: hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case types.ChecksumAlgorithmCrc64nvme: + hasher = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))) + case types.ChecksumAlgorithmXxhash64: + hasher = xxhash.New() + case types.ChecksumAlgorithmXxhash3: + hasher = xxh3.New() + case types.ChecksumAlgorithmXxhash128: + hasher = xxh3.New128() default: return nil, fmt.Errorf("unsupported hash algorithm: %s", algo) } @@ -2027,6 +2427,28 @@ func NewHasher(algo types.ChecksumAlgorithm) (hash.Hash, error) { return hasher, nil } +func wrongChecksumForAlgorithm(algo types.ChecksumAlgorithm) (string, error) { + var size int + switch algo { + case types.ChecksumAlgorithmCrc32, types.ChecksumAlgorithmCrc32c: + size = 4 + case types.ChecksumAlgorithmCrc64nvme, types.ChecksumAlgorithmXxhash64, types.ChecksumAlgorithmXxhash3: + size = 8 + case types.ChecksumAlgorithmMd5, types.ChecksumAlgorithmXxhash128: + size = 16 + case types.ChecksumAlgorithmSha1: + size = 20 + case types.ChecksumAlgorithmSha256: + size = 32 + case types.ChecksumAlgorithmSha512: + size = 64 + default: + return "", fmt.Errorf("unsupported hash algorithm: %s", algo) + } + + return base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0xff}, size)), nil +} + func processCompositeChecksum(hasher hash.Hash, checksum string) error { data, err := base64.StdEncoding.DecodeString(checksum) if err != nil {