Merge pull request #1894 from versity/sis/getobject-directory-object-checksum

feat: adds checksums for directory objects in posix
This commit is contained in:
Ben McClelland
2026-02-28 09:39:51 -08:00
committed by GitHub
4 changed files with 322 additions and 84 deletions
+129 -84
View File
@@ -3052,6 +3052,24 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
}, nil
}
// getEmptyChecksumValue returns the base64-encoded checksum
// for an empty payload for the given algorithm defaulting to crc64nvme
func getEmptyChecksumValue(algo types.ChecksumAlgorithm) string {
switch algo {
case types.ChecksumAlgorithmCrc32:
return "AAAAAA=="
case types.ChecksumAlgorithmCrc32c:
return "AAAAAA=="
case types.ChecksumAlgorithmSha1:
return "2jmj7l5rSw0yVb/vlWAYkK/YBwk="
case types.ChecksumAlgorithmSha256:
return "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
default:
// default to crc64nvme
return "AAAAAAAAAAA="
}
}
func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
release, err := p.acquireActionSlot(ctx)
if err != nil {
@@ -3104,6 +3122,39 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
if po.ContentLength != nil {
contentLength = *po.ContentLength
}
chRdr, chunkUpload := po.Body.(middlewares.ChecksumReader)
isTrailingChecksum := chunkUpload && chRdr.Algorithm() != ""
checksumAlgorithm := po.ChecksumAlgorithm
checksumValue := ""
if isTrailingChecksum {
checksumAlgorithm = types.ChecksumAlgorithm(strings.ToUpper(chRdr.Algorithm()))
checksumValue = chRdr.Checksum()
} else {
hashConfigs := []hashConfig{
{po.ChecksumCRC32, utils.HashTypeCRC32},
{po.ChecksumCRC32C, utils.HashTypeCRC32C},
{po.ChecksumSHA1, utils.HashTypeSha1},
{po.ChecksumSHA256, utils.HashTypeSha256},
{po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME},
}
for _, config := range hashConfigs {
if config.value != nil {
checksumAlgorithm = types.ChecksumAlgorithm(strings.ToUpper(string(config.hashType)))
checksumValue = *config.value
break
}
}
if checksumAlgorithm == "" {
// default to crc64nvme
checksumAlgorithm = types.ChecksumAlgorithmCrc64nvme
}
}
if strings.HasSuffix(*po.Key, "/") {
// object is directory
if contentLength != 0 {
@@ -3143,10 +3194,46 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err)
}
expectedSum := getEmptyChecksumValue(checksumAlgorithm)
if checksumValue != "" && expectedSum != checksumValue {
return s3response.PutObjectOutput{}, s3err.GetChecksumBadDigestErr(checksumAlgorithm)
}
// set empty checksum
checksum := s3response.Checksum{
Type: types.ChecksumTypeFullObject,
Algorithm: checksumAlgorithm,
}
// Store the calculated checksum in the object metadata
switch checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = &expectedSum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = &expectedSum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = &expectedSum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = &expectedSum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = &expectedSum
}
err = p.storeChecksums(nil, *po.Bucket, *po.Key, checksum)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
}
// for directory object no version is created
return s3response.PutObjectOutput{
ETag: emptyMD5,
Size: &contentLength,
ETag: emptyMD5,
Size: &contentLength,
ChecksumType: checksum.Type,
ChecksumCRC32: checksum.CRC32,
ChecksumCRC32C: checksum.CRC32C,
ChecksumCRC64NVME: checksum.CRC64NVME,
ChecksumSHA1: checksum.SHA1,
ChecksumSHA256: checksum.SHA256,
}, nil
}
@@ -3208,53 +3295,13 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
rdr := io.TeeReader(po.Body, hash)
var hashRdr *utils.HashReader
chRdr, chunkUpload := po.Body.(middlewares.ChecksumReader)
isTrailingChecksum := chunkUpload && chRdr.Algorithm() != ""
if !isTrailingChecksum {
hashConfigs := []hashConfig{
{po.ChecksumCRC32, utils.HashTypeCRC32},
{po.ChecksumCRC32C, utils.HashTypeCRC32C},
{po.ChecksumSHA1, utils.HashTypeSha1},
{po.ChecksumSHA256, utils.HashTypeSha256},
{po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME},
hashRdr, err = utils.NewHashReader(rdr, checksumValue, utils.HashType(strings.ToLower(string(checksumAlgorithm))))
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err)
}
for _, config := range hashConfigs {
if config.value != nil {
hashRdr, err = utils.NewHashReader(rdr, *config.value, config.hashType)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err)
}
rdr = hashRdr
}
}
// If only the checksum algorithm is provided register
// a new HashReader to calculate the object checksum
// This can never happen with PutObject direct call
// it's there for CopyObject to add a new checksum
if hashRdr == nil && po.ChecksumAlgorithm != "" {
hashRdr, err = utils.NewHashReader(rdr, "", utils.HashType(strings.ToLower(string(po.ChecksumAlgorithm))))
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err)
}
rdr = hashRdr
}
if hashRdr == nil {
// if no precalculated checksum or sdk checksum algorithm is provided
// and no streaming upload has checksum, default to crc64nvme
hashRdr, err = utils.NewHashReader(rdr, "", utils.HashTypeCRC64NVME)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err)
}
rdr = hashRdr
}
rdr = hashRdr
}
_, err = io.Copy(f, rdr)
@@ -3304,43 +3351,35 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
}
}
var chAlgo utils.HashType
var sum string
if isTrailingChecksum {
chAlgo = utils.HashType(chRdr.Algorithm())
sum = chRdr.Checksum()
} else if hashRdr != nil {
chAlgo = hashRdr.Type()
} else {
sum = hashRdr.Sum()
}
checksum := s3response.Checksum{}
checksum := s3response.Checksum{
Type: types.ChecksumTypeFullObject,
Algorithm: checksumAlgorithm,
}
// Store the calculated checksum in the object metadata
if sum != "" {
checksum.Type = types.ChecksumTypeFullObject
switch chAlgo {
case utils.HashTypeCRC32:
checksum.CRC32 = &sum
checksum.Algorithm = types.ChecksumAlgorithmCrc32
case utils.HashTypeCRC32C:
checksum.CRC32C = &sum
checksum.Algorithm = types.ChecksumAlgorithmCrc32c
case utils.HashTypeSha1:
checksum.SHA1 = &sum
checksum.Algorithm = types.ChecksumAlgorithmSha1
case utils.HashTypeSha256:
checksum.SHA256 = &sum
checksum.Algorithm = types.ChecksumAlgorithmSha256
case utils.HashTypeCRC64NVME:
checksum.CRC64NVME = &sum
checksum.Algorithm = types.ChecksumAlgorithmCrc64nvme
}
err := p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
}
switch checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
checksum.CRC32 = &sum
case types.ChecksumAlgorithmCrc32c:
checksum.CRC32C = &sum
case types.ChecksumAlgorithmSha1:
checksum.SHA1 = &sum
case types.ChecksumAlgorithmSha256:
checksum.SHA256 = &sum
case types.ChecksumAlgorithmCrc64nvme:
checksum.CRC64NVME = &sum
}
err = p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum)
if err != nil {
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
}
err = p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, etagkey, []byte(etag))
@@ -4000,8 +4039,22 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
tagCount = &tgCount
}
var checksums s3response.Checksum
if input.ChecksumMode == types.ChecksumModeEnabled {
checksums, err = p.retrieveChecksums(nil, bucket, object)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object checksums: %w", err)
}
}
var length int64 = 0
return &s3.GetObjectOutput{
ChecksumCRC32: checksums.CRC32,
ChecksumCRC32C: checksums.CRC32C,
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumType: checksums.Type,
AcceptRanges: backend.GetPtrFromString("bytes"),
ContentLength: &length,
ContentEncoding: objMeta.ContentEncoding,
@@ -4081,16 +4134,12 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
}
var checksums s3response.Checksum
var cType types.ChecksumType
// Skip the checksums retreival if object isn't requested fully
if input.ChecksumMode == types.ChecksumModeEnabled && length-startOffset == objSize {
checksums, err = p.retrieveChecksums(f, bucket, object)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object checksums: %w", err)
}
if checksums.Type != "" {
cType = checksums.Type
}
}
// using an os.File allows zero-copy sendfile via io.Copy(os.File, net.Conn)
@@ -4122,7 +4171,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumType: cType,
ChecksumType: checksums.Type,
}, nil
}
@@ -4290,15 +4339,11 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
}
var checksums s3response.Checksum
var cType types.ChecksumType
if input.ChecksumMode == types.ChecksumModeEnabled {
checksums, err = p.retrieveChecksums(nil, bucket, object)
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object checksums: %w", err)
}
if checksums.Type != "" {
cType = checksums.Type
}
}
var tagCount *int32
@@ -4334,7 +4379,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
ChecksumSHA1: checksums.SHA1,
ChecksumSHA256: checksums.SHA256,
ChecksumCRC64NVME: checksums.CRC64NVME,
ChecksumType: cType,
ChecksumType: checksums.Type,
TagCount: tagCount,
}, nil
}
+84
View File
@@ -308,6 +308,90 @@ func GetObject_checksums(s *S3Conf) error {
})
}
func GetObject_dir_object_checksum(s *S3Conf) error {
testName := "GetObject_dir_object_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
for i, obj := range []struct {
key string
expectedSum string
checksumAlgo types.ChecksumAlgorithm
}{
{
key: "obj-1/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32,
},
{
key: "obj-2/",
expectedSum: "AAAAAA==",
checksumAlgo: types.ChecksumAlgorithmCrc32c,
},
{
key: "obj-3/",
expectedSum: "AAAAAAAAAAA=",
checksumAlgo: types.ChecksumAlgorithmCrc64nvme,
},
{
key: "obj-4/",
expectedSum: "2jmj7l5rSw0yVb/vlWAYkK/YBwk=",
checksumAlgo: types.ChecksumAlgorithmSha1,
},
{
key: "obj-5/",
expectedSum: "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
checksumAlgo: types.ChecksumAlgorithmSha256,
},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj.key,
ChecksumAlgorithm: obj.checksumAlgo,
})
cancel()
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &obj.key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
if err != nil {
return fmt.Errorf("test %v failed: %w", i+1, err)
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("test %v failed: expected the %v object checksum type to be %v, instaed got %v",
i+1, obj.key, types.ChecksumTypeFullObject, res.ChecksumType)
}
var gotSum *string
switch obj.checksumAlgo {
case types.ChecksumAlgorithmCrc32:
gotSum = res.ChecksumCRC32
case types.ChecksumAlgorithmCrc32c:
gotSum = res.ChecksumCRC32C
case types.ChecksumAlgorithmCrc64nvme:
gotSum = res.ChecksumCRC64NVME
case types.ChecksumAlgorithmSha1:
gotSum = res.ChecksumSHA1
case types.ChecksumAlgorithmSha256:
gotSum = res.ChecksumSHA256
}
if getString(gotSum) != obj.expectedSum {
return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, obj.checksumAlgo, obj.expectedSum, getString(gotSum))
}
}
return nil
})
}
func GetObject_large_object(s *S3Conf) error {
testName := "GetObject_large_object"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+103
View File
@@ -698,6 +698,7 @@ func PutObject_incorrect_checksums(s *S3Conf) error {
testName := "PutObject_incorrect_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
dirObj := "dir-object/"
for i, el := range []struct {
algo types.ChecksumAlgorithm
@@ -728,6 +729,7 @@ func PutObject_incorrect_checksums(s *S3Conf) error {
crc64nvme: getPtr("sV264W+gYBI="),
},
} {
// test for file object
_, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
@@ -740,6 +742,22 @@ func PutObject_incorrect_checksums(s *S3Conf) error {
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
return err
}
// test for directory object
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &bucket,
Key: &dirObj,
ChecksumCRC32: el.crc32,
ChecksumCRC32C: el.crc32c,
ChecksumSHA1: el.sha1,
ChecksumSHA256: el.sha256,
ChecksumCRC64NVME: el.crc64nvme,
})
cancel()
if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil {
return err
}
}
return nil
@@ -786,6 +804,91 @@ func PutObject_default_checksum(s *S3Conf) error {
})
}
func PutObject_dir_object_default_checksum(s *S3Conf) error {
testName := "PutObject_dir_object_default_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "dir/obj/"
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, func(o *s3.Options) {
o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset
})
cancel()
if err != nil {
return err
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the object checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType)
}
expectedcrc64nvme := "AAAAAAAAAAA="
if getString(res.ChecksumCRC64NVME) != expectedcrc64nvme {
return fmt.Errorf("expected the crc64nvme checksum to be %s, instead got %s", expectedcrc64nvme, getString(res.ChecksumCRC64NVME))
}
return nil
})
}
func PutObject_dir_object_checksums_success(s *S3Conf) error {
testName := "PutObject_dir_object_checksums_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
for i, test := range []struct {
algo types.ChecksumAlgorithm
checksumValue string
}{
{types.ChecksumAlgorithmCrc32, "AAAAAA=="},
{types.ChecksumAlgorithmCrc32c, "AAAAAA=="},
{types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="},
{types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="},
{types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &bucket,
Key: getPtr(fmt.Sprintf("obj-%v/", i)),
ChecksumAlgorithm: test.algo,
})
cancel()
if err != nil {
return err
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType)
}
switch test.algo {
case types.ChecksumAlgorithmCrc32:
if getString(res.ChecksumCRC32) != test.checksumValue {
return fmt.Errorf("expected the crc32 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32))
}
case types.ChecksumAlgorithmCrc32c:
if getString(res.ChecksumCRC32C) != test.checksumValue {
return fmt.Errorf("expected the crc32c checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32C))
}
case types.ChecksumAlgorithmSha1:
if getString(res.ChecksumSHA1) != test.checksumValue {
return fmt.Errorf("expected the sha1 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA1))
}
case types.ChecksumAlgorithmSha256:
if getString(res.ChecksumSHA256) != test.checksumValue {
return fmt.Errorf("expected the sha256 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA256))
}
case types.ChecksumAlgorithmCrc64nvme:
if getString(res.ChecksumCRC64NVME) != test.checksumValue {
return fmt.Errorf("expected the crc64nvme checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC64NVME))
}
}
}
return nil
})
}
func PutObject_checksums_success(s *S3Conf) error {
testName := "PutObject_checksums_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+6
View File
@@ -178,7 +178,9 @@ func TestPutObject(ts *TestState) {
ts.Run(PutObject_invalid_checksum_header)
ts.Run(PutObject_incorrect_checksums)
ts.Run(PutObject_default_checksum)
ts.Run(PutObject_dir_object_default_checksum)
ts.Run(PutObject_checksums_success)
ts.Run(PutObject_dir_object_checksums_success)
// azure applies some encoding mechanisms.
ts.Run(PutObject_false_negative_object_names)
// azure doesn't support these metadata characters
@@ -239,6 +241,7 @@ func TestGetObject(ts *TestState) {
//TODO: remove the condition after implementing checksums in azure
if !ts.conf.azureTests {
ts.Run(GetObject_checksums)
ts.Run(GetObject_dir_object_checksum)
}
ts.Run(GetObject_success)
ts.Run(GetObject_directory_success)
@@ -1247,7 +1250,9 @@ func GetIntTests() IntTests {
"PutObject_invalid_checksum_header": PutObject_invalid_checksum_header,
"PutObject_incorrect_checksums": PutObject_incorrect_checksums,
"PutObject_default_checksum": PutObject_default_checksum,
"PutObject_dir_object_default_checksum": PutObject_dir_object_default_checksum,
"PutObject_checksums_success": PutObject_checksums_success,
"PutObject_dir_object_checksums_success": PutObject_dir_object_checksums_success,
"PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data,
"PresignedAuth_Put_GetObject_with_UTF8_chars": PresignedAuth_Put_GetObject_with_UTF8_chars,
"PresignedAuth_UploadPart": PresignedAuth_UploadPart,
@@ -1346,6 +1351,7 @@ func GetIntTests() IntTests {
"GetObject_large_object": GetObject_large_object,
"GetObject_conditional_reads": GetObject_conditional_reads,
"GetObject_checksums": GetObject_checksums,
"GetObject_dir_object_checksum": GetObject_dir_object_checksum,
"GetObject_success": GetObject_success,
"GetObject_directory_success": GetObject_directory_success,
"GetObject_by_range_resp_status": GetObject_by_range_resp_status,