From 24364754fd38dc27531392fd53bac00b7c6a9a71 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Wed, 25 Feb 2026 02:19:02 +0400 Subject: [PATCH] feat: adds checksums for directory objects in posix Add data-integrity checksum support in `PutObject` in the POSIX backend for directory objects. Since the only way to upload a directory object is via `PutObject`, this logic validates and stores the checksum of the empty payload. Support for `GetObject` has also been added to retrieve and return directory-object checksums. --- backend/posix/posix.go | 213 +++++++++++++++++++------------ tests/integration/GetObject.go | 84 ++++++++++++ tests/integration/PutObject.go | 103 +++++++++++++++ tests/integration/group-tests.go | 6 + 4 files changed, 322 insertions(+), 84 deletions(-) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index a0b14656..5750711f 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -3041,6 +3041,24 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) }, nil } +// getEmptyChecksumValue returns the base64-encoded checksum +// for an empty payload for the given algorithm defaulting to crc64nvme +func getEmptyChecksumValue(algo types.ChecksumAlgorithm) string { + switch algo { + case types.ChecksumAlgorithmCrc32: + return "AAAAAA==" + case types.ChecksumAlgorithmCrc32c: + return "AAAAAA==" + case types.ChecksumAlgorithmSha1: + return "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" + case types.ChecksumAlgorithmSha256: + return "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + default: + // default to crc64nvme + return "AAAAAAAAAAA=" + } +} + func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3response.PutObjectOutput, error) { release, err := p.acquireActionSlot(ctx) if err != nil { @@ -3093,6 +3111,39 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje if po.ContentLength != nil { contentLength = *po.ContentLength } + + chRdr, chunkUpload := po.Body.(middlewares.ChecksumReader) + isTrailingChecksum := chunkUpload && chRdr.Algorithm() != "" + + checksumAlgorithm := po.ChecksumAlgorithm + checksumValue := "" + + if isTrailingChecksum { + checksumAlgorithm = types.ChecksumAlgorithm(strings.ToUpper(chRdr.Algorithm())) + checksumValue = chRdr.Checksum() + } else { + hashConfigs := []hashConfig{ + {po.ChecksumCRC32, utils.HashTypeCRC32}, + {po.ChecksumCRC32C, utils.HashTypeCRC32C}, + {po.ChecksumSHA1, utils.HashTypeSha1}, + {po.ChecksumSHA256, utils.HashTypeSha256}, + {po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME}, + } + + for _, config := range hashConfigs { + if config.value != nil { + checksumAlgorithm = types.ChecksumAlgorithm(strings.ToUpper(string(config.hashType))) + checksumValue = *config.value + break + } + } + + if checksumAlgorithm == "" { + // default to crc64nvme + checksumAlgorithm = types.ChecksumAlgorithmCrc64nvme + } + } + if strings.HasSuffix(*po.Key, "/") { // object is directory if contentLength != 0 { @@ -3132,10 +3183,46 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err) } + expectedSum := getEmptyChecksumValue(checksumAlgorithm) + if checksumValue != "" && expectedSum != checksumValue { + return s3response.PutObjectOutput{}, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + + // set empty checksum + checksum := s3response.Checksum{ + Type: types.ChecksumTypeFullObject, + Algorithm: checksumAlgorithm, + } + + // Store the calculated checksum in the object metadata + switch checksumAlgorithm { + case types.ChecksumAlgorithmCrc32: + checksum.CRC32 = &expectedSum + case types.ChecksumAlgorithmCrc32c: + checksum.CRC32C = &expectedSum + case types.ChecksumAlgorithmSha1: + checksum.SHA1 = &expectedSum + case types.ChecksumAlgorithmSha256: + checksum.SHA256 = &expectedSum + case types.ChecksumAlgorithmCrc64nvme: + checksum.CRC64NVME = &expectedSum + } + + err = p.storeChecksums(nil, *po.Bucket, *po.Key, checksum) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) + } + // for directory object no version is created return s3response.PutObjectOutput{ - ETag: emptyMD5, - Size: &contentLength, + ETag: emptyMD5, + Size: &contentLength, + ChecksumType: checksum.Type, + ChecksumCRC32: checksum.CRC32, + ChecksumCRC32C: checksum.CRC32C, + ChecksumCRC64NVME: checksum.CRC64NVME, + ChecksumSHA1: checksum.SHA1, + ChecksumSHA256: checksum.SHA256, }, nil } @@ -3197,53 +3284,13 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje rdr := io.TeeReader(po.Body, hash) var hashRdr *utils.HashReader - - chRdr, chunkUpload := po.Body.(middlewares.ChecksumReader) - isTrailingChecksum := chunkUpload && chRdr.Algorithm() != "" - if !isTrailingChecksum { - hashConfigs := []hashConfig{ - {po.ChecksumCRC32, utils.HashTypeCRC32}, - {po.ChecksumCRC32C, utils.HashTypeCRC32C}, - {po.ChecksumSHA1, utils.HashTypeSha1}, - {po.ChecksumSHA256, utils.HashTypeSha256}, - {po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME}, + hashRdr, err = utils.NewHashReader(rdr, checksumValue, utils.HashType(strings.ToLower(string(checksumAlgorithm)))) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) } - for _, config := range hashConfigs { - if config.value != nil { - hashRdr, err = utils.NewHashReader(rdr, *config.value, config.hashType) - if err != nil { - return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) - } - - rdr = hashRdr - } - } - - // If only the checksum algorithm is provided register - // a new HashReader to calculate the object checksum - // This can never happen with PutObject direct call - // it's there for CopyObject to add a new checksum - if hashRdr == nil && po.ChecksumAlgorithm != "" { - hashRdr, err = utils.NewHashReader(rdr, "", utils.HashType(strings.ToLower(string(po.ChecksumAlgorithm)))) - if err != nil { - return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) - } - - rdr = hashRdr - } - - if hashRdr == nil { - // if no precalculated checksum or sdk checksum algorithm is provided - // and no streaming upload has checksum, default to crc64nvme - hashRdr, err = utils.NewHashReader(rdr, "", utils.HashTypeCRC64NVME) - if err != nil { - return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) - } - - rdr = hashRdr - } + rdr = hashRdr } _, err = io.Copy(f, rdr) @@ -3293,43 +3340,35 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } } - var chAlgo utils.HashType var sum string if isTrailingChecksum { - chAlgo = utils.HashType(chRdr.Algorithm()) sum = chRdr.Checksum() - } else if hashRdr != nil { - chAlgo = hashRdr.Type() + } else { sum = hashRdr.Sum() } - checksum := s3response.Checksum{} + checksum := s3response.Checksum{ + Type: types.ChecksumTypeFullObject, + Algorithm: checksumAlgorithm, + } // Store the calculated checksum in the object metadata - if sum != "" { - checksum.Type = types.ChecksumTypeFullObject - switch chAlgo { - case utils.HashTypeCRC32: - checksum.CRC32 = &sum - checksum.Algorithm = types.ChecksumAlgorithmCrc32 - case utils.HashTypeCRC32C: - checksum.CRC32C = &sum - checksum.Algorithm = types.ChecksumAlgorithmCrc32c - case utils.HashTypeSha1: - checksum.SHA1 = &sum - checksum.Algorithm = types.ChecksumAlgorithmSha1 - case utils.HashTypeSha256: - checksum.SHA256 = &sum - checksum.Algorithm = types.ChecksumAlgorithmSha256 - case utils.HashTypeCRC64NVME: - checksum.CRC64NVME = &sum - checksum.Algorithm = types.ChecksumAlgorithmCrc64nvme - } - err := p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum) - if err != nil { - return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) - } + switch checksumAlgorithm { + case types.ChecksumAlgorithmCrc32: + checksum.CRC32 = &sum + case types.ChecksumAlgorithmCrc32c: + checksum.CRC32C = &sum + case types.ChecksumAlgorithmSha1: + checksum.SHA1 = &sum + case types.ChecksumAlgorithmSha256: + checksum.SHA256 = &sum + case types.ChecksumAlgorithmCrc64nvme: + checksum.CRC64NVME = &sum + } + err = p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) } err = p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, etagkey, []byte(etag)) @@ -3989,8 +4028,22 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge tagCount = &tgCount } + var checksums s3response.Checksum + if input.ChecksumMode == types.ChecksumModeEnabled { + checksums, err = p.retrieveChecksums(nil, bucket, object) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get object checksums: %w", err) + } + } + var length int64 = 0 return &s3.GetObjectOutput{ + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumType: checksums.Type, AcceptRanges: backend.GetPtrFromString("bytes"), ContentLength: &length, ContentEncoding: objMeta.ContentEncoding, @@ -4070,16 +4123,12 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge } var checksums s3response.Checksum - var cType types.ChecksumType // Skip the checksums retreival if object isn't requested fully if input.ChecksumMode == types.ChecksumModeEnabled && length-startOffset == objSize { checksums, err = p.retrieveChecksums(f, bucket, object) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return nil, fmt.Errorf("get object checksums: %w", err) } - if checksums.Type != "" { - cType = checksums.Type - } } // using an os.File allows zero-copy sendfile via io.Copy(os.File, net.Conn) @@ -4111,7 +4160,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge ChecksumSHA1: checksums.SHA1, ChecksumSHA256: checksums.SHA256, ChecksumCRC64NVME: checksums.CRC64NVME, - ChecksumType: cType, + ChecksumType: checksums.Type, }, nil } @@ -4279,15 +4328,11 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } var checksums s3response.Checksum - var cType types.ChecksumType if input.ChecksumMode == types.ChecksumModeEnabled { checksums, err = p.retrieveChecksums(nil, bucket, object) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return nil, fmt.Errorf("get object checksums: %w", err) } - if checksums.Type != "" { - cType = checksums.Type - } } var tagCount *int32 @@ -4323,7 +4368,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ChecksumSHA1: checksums.SHA1, ChecksumSHA256: checksums.SHA256, ChecksumCRC64NVME: checksums.CRC64NVME, - ChecksumType: cType, + ChecksumType: checksums.Type, TagCount: tagCount, }, nil } diff --git a/tests/integration/GetObject.go b/tests/integration/GetObject.go index 67a0017e..6ee11397 100644 --- a/tests/integration/GetObject.go +++ b/tests/integration/GetObject.go @@ -308,6 +308,90 @@ func GetObject_checksums(s *S3Conf) error { }) } +func GetObject_dir_object_checksum(s *S3Conf) error { + testName := "GetObject_dir_object_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for i, obj := range []struct { + key string + expectedSum string + checksumAlgo types.ChecksumAlgorithm + }{ + { + key: "obj-1/", + expectedSum: "AAAAAA==", + checksumAlgo: types.ChecksumAlgorithmCrc32, + }, + { + key: "obj-2/", + expectedSum: "AAAAAA==", + checksumAlgo: types.ChecksumAlgorithmCrc32c, + }, + { + key: "obj-3/", + expectedSum: "AAAAAAAAAAA=", + checksumAlgo: types.ChecksumAlgorithmCrc64nvme, + }, + { + key: "obj-4/", + expectedSum: "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", + checksumAlgo: types.ChecksumAlgorithmSha1, + }, + { + key: "obj-5/", + expectedSum: "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", + checksumAlgo: types.ChecksumAlgorithmSha256, + }, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj.key, + ChecksumAlgorithm: obj.checksumAlgo, + }) + cancel() + if err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj.key, + ChecksumMode: types.ChecksumModeEnabled, + }) + cancel() + if err != nil { + return fmt.Errorf("test %v failed: %w", i+1, err) + } + + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("test %v failed: expected the %v object checksum type to be %v, instaed got %v", + i+1, obj.key, types.ChecksumTypeFullObject, res.ChecksumType) + } + + var gotSum *string + switch obj.checksumAlgo { + case types.ChecksumAlgorithmCrc32: + gotSum = res.ChecksumCRC32 + case types.ChecksumAlgorithmCrc32c: + gotSum = res.ChecksumCRC32C + case types.ChecksumAlgorithmCrc64nvme: + gotSum = res.ChecksumCRC64NVME + case types.ChecksumAlgorithmSha1: + gotSum = res.ChecksumSHA1 + case types.ChecksumAlgorithmSha256: + gotSum = res.ChecksumSHA256 + } + + if getString(gotSum) != obj.expectedSum { + return fmt.Errorf("test %v failed: expected the object %s to be %s, instead got %s", i+1, obj.checksumAlgo, obj.expectedSum, getString(gotSum)) + } + } + + return nil + }) +} + func GetObject_large_object(s *S3Conf) error { testName := "GetObject_large_object" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/PutObject.go b/tests/integration/PutObject.go index 3618ca8c..6b848f0a 100644 --- a/tests/integration/PutObject.go +++ b/tests/integration/PutObject.go @@ -698,6 +698,7 @@ func PutObject_incorrect_checksums(s *S3Conf) error { testName := "PutObject_incorrect_checksums" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" + dirObj := "dir-object/" for i, el := range []struct { algo types.ChecksumAlgorithm @@ -728,6 +729,7 @@ func PutObject_incorrect_checksums(s *S3Conf) error { crc64nvme: getPtr("sV264W+gYBI="), }, } { + // test for file object _, err := putObjectWithData(int64(i*100), &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, @@ -740,6 +742,22 @@ func PutObject_incorrect_checksums(s *S3Conf) error { if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { return err } + + // test for directory object + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &dirObj, + ChecksumCRC32: el.crc32, + ChecksumCRC32C: el.crc32c, + ChecksumSHA1: el.sha1, + ChecksumSHA256: el.sha256, + ChecksumCRC64NVME: el.crc64nvme, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + return err + } } return nil @@ -786,6 +804,91 @@ func PutObject_default_checksum(s *S3Conf) error { }) } +func PutObject_dir_object_default_checksum(s *S3Conf) error { + testName := "PutObject_dir_object_default_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "dir/obj/" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, func(o *s3.Options) { + o.RequestChecksumCalculation = aws.RequestChecksumCalculationUnset + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the object checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType) + } + expectedcrc64nvme := "AAAAAAAAAAA=" + if getString(res.ChecksumCRC64NVME) != expectedcrc64nvme { + return fmt.Errorf("expected the crc64nvme checksum to be %s, instead got %s", expectedcrc64nvme, getString(res.ChecksumCRC64NVME)) + } + + return nil + }) +} + +func PutObject_dir_object_checksums_success(s *S3Conf) error { + testName := "PutObject_dir_object_checksums_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for i, test := range []struct { + algo types.ChecksumAlgorithm + checksumValue string + }{ + {types.ChecksumAlgorithmCrc32, "AAAAAA=="}, + {types.ChecksumAlgorithmCrc32c, "AAAAAA=="}, + {types.ChecksumAlgorithmCrc64nvme, "AAAAAAAAAAA="}, + {types.ChecksumAlgorithmSha1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="}, + {types.ChecksumAlgorithmSha256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="}, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: getPtr(fmt.Sprintf("obj-%v/", i)), + ChecksumAlgorithm: test.algo, + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the checksum type to be %s, instead got %s", types.ChecksumTypeFullObject, res.ChecksumType) + } + + switch test.algo { + case types.ChecksumAlgorithmCrc32: + if getString(res.ChecksumCRC32) != test.checksumValue { + return fmt.Errorf("expected the crc32 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32)) + } + case types.ChecksumAlgorithmCrc32c: + if getString(res.ChecksumCRC32C) != test.checksumValue { + return fmt.Errorf("expected the crc32c checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC32C)) + } + case types.ChecksumAlgorithmSha1: + if getString(res.ChecksumSHA1) != test.checksumValue { + return fmt.Errorf("expected the sha1 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA1)) + } + case types.ChecksumAlgorithmSha256: + if getString(res.ChecksumSHA256) != test.checksumValue { + return fmt.Errorf("expected the sha256 checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumSHA256)) + } + case types.ChecksumAlgorithmCrc64nvme: + if getString(res.ChecksumCRC64NVME) != test.checksumValue { + return fmt.Errorf("expected the crc64nvme checksum value to be %s, instead got %s", test.checksumValue, getString(res.ChecksumCRC64NVME)) + } + } + } + + return nil + }) +} + func PutObject_checksums_success(s *S3Conf) error { testName := "PutObject_checksums_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index f9863afc..f3344a3c 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -178,7 +178,9 @@ func TestPutObject(ts *TestState) { ts.Run(PutObject_invalid_checksum_header) ts.Run(PutObject_incorrect_checksums) ts.Run(PutObject_default_checksum) + ts.Run(PutObject_dir_object_default_checksum) ts.Run(PutObject_checksums_success) + ts.Run(PutObject_dir_object_checksums_success) // azure applies some encoding mechanisms. ts.Run(PutObject_false_negative_object_names) // azure doesn't support these metadata characters @@ -239,6 +241,7 @@ func TestGetObject(ts *TestState) { //TODO: remove the condition after implementing checksums in azure if !ts.conf.azureTests { ts.Run(GetObject_checksums) + ts.Run(GetObject_dir_object_checksum) } ts.Run(GetObject_success) ts.Run(GetObject_directory_success) @@ -1237,7 +1240,9 @@ func GetIntTests() IntTests { "PutObject_invalid_checksum_header": PutObject_invalid_checksum_header, "PutObject_incorrect_checksums": PutObject_incorrect_checksums, "PutObject_default_checksum": PutObject_default_checksum, + "PutObject_dir_object_default_checksum": PutObject_dir_object_default_checksum, "PutObject_checksums_success": PutObject_checksums_success, + "PutObject_dir_object_checksums_success": PutObject_dir_object_checksums_success, "PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data, "PresignedAuth_Put_GetObject_with_UTF8_chars": PresignedAuth_Put_GetObject_with_UTF8_chars, "PresignedAuth_UploadPart": PresignedAuth_UploadPart, @@ -1336,6 +1341,7 @@ func GetIntTests() IntTests { "GetObject_large_object": GetObject_large_object, "GetObject_conditional_reads": GetObject_conditional_reads, "GetObject_checksums": GetObject_checksums, + "GetObject_dir_object_checksum": GetObject_dir_object_checksum, "GetObject_success": GetObject_success, "GetObject_directory_success": GetObject_directory_success, "GetObject_by_range_resp_status": GetObject_by_range_resp_status,