From fcafb57abc5080c7d5631983c98b01a8607fdd7b Mon Sep 17 00:00:00 2001 From: niksis02 Date: Thu, 27 Feb 2025 00:24:38 +0400 Subject: [PATCH] feat: Implements checksums for ListObjectVersions and CopyObject actions. --- backend/posix/posix.go | 62 ++++++++++++++++++++++++-------- tests/integration/group-tests.go | 2 ++ tests/integration/tests.go | 52 +++++++++++++++++++++------ tests/integration/utils.go | 61 ++++++++++++++++++++++++++++--- 4 files changed, 149 insertions(+), 28 deletions(-) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index d42f1366..52a5c152 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -915,14 +915,22 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { Key: &path, }) } else { + // Retreive checksum + checksum, err := p.retrieveChecksums(nil, bucket, path) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get checksum: %w", err) + } + objects = append(objects, types.ObjectVersion{ - ETag: &etag, - Key: &path, - LastModified: backend.GetTimePtr(fi.ModTime()), - Size: &size, - VersionId: &versionId, - IsLatest: getBoolPtr(true), - StorageClass: types.ObjectVersionStorageClassStandard, + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(fi.ModTime()), + Size: &size, + VersionId: &versionId, + IsLatest: getBoolPtr(true), + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{checksum.Algorithm}, + ChecksumType: checksum.Type, }) } @@ -998,6 +1006,12 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { // so this will just set etag to "" if its not already set etag := string(etagBytes) size := nf.Size() + // Retreive checksum + checksum, err := p.retrieveChecksums(nil, versionPath, nullVersionId) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get checksum: %w", err) + } + nullVersionIdObj = &types.ObjectVersion{ ETag: &etag, Key: &path, @@ -1006,6 +1020,10 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { VersionId: backend.GetPtrFromString("null"), IsLatest: getBoolPtr(false), StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{ + checksum.Algorithm, + }, + ChecksumType: checksum.Type, } } } @@ -1110,14 +1128,21 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { IsLatest: getBoolPtr(false), }) } else { + // Retreive checksum + checksum, err := p.retrieveChecksums(nil, versionPath, versionId) + if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { + return nil, fmt.Errorf("get checksum: %w", err) + } objects = append(objects, types.ObjectVersion{ - ETag: &etag, - Key: &path, - LastModified: backend.GetTimePtr(f.ModTime()), - Size: &size, - VersionId: &versionId, - IsLatest: getBoolPtr(false), - StorageClass: types.ObjectVersionStorageClassStandard, + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(f.ModTime()), + Size: &size, + VersionId: &versionId, + IsLatest: getBoolPtr(false), + StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumAlgorithm: []types.ChecksumAlgorithm{checksum.Algorithm}, + ChecksumType: checksum.Type, }) } @@ -3855,6 +3880,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. var sha1 *string var sha256 *string var crc64nvme *string + var chType types.ChecksumType dstObjdPath := filepath.Join(dstBucket, dstObject) if dstObjdPath == objPath { @@ -3882,6 +3908,8 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. return nil, fmt.Errorf("get obj checksums: %w", err) } + chType = checksums.Type + if input.ChecksumAlgorithm != "" { // If a different checksum algorith is specified // first caclculate and store the checksum @@ -3923,6 +3951,10 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. crc64nvme = &sum } + // If a new checksum is calculated, the checksum type + // should be FULL_OBJECT + chType = types.ChecksumTypeFullObject + err = p.storeChecksums(f, dstBucket, dstObject, checksums) if err != nil { return nil, fmt.Errorf("store checksum: %w", err) @@ -3970,6 +4002,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. sha1 = res.ChecksumSHA1 sha256 = res.ChecksumSHA256 crc64nvme = res.ChecksumCRC64NVME + chType = res.ChecksumType } fi, err = os.Stat(dstObjdPath) @@ -3986,6 +4019,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. ChecksumSHA1: sha1, ChecksumSHA256: sha256, ChecksumCRC64NVME: crc64nvme, + ChecksumType: chType, }, VersionId: version, CopySourceVersionId: &srcVersionId, diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index e97ac448..a81de847 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -703,6 +703,7 @@ func TestVersioning(s *S3Conf) { ListObjectVersions_with_delete_markers(s) ListObjectVersions_containing_null_versionId_obj(s) ListObjectVersions_single_null_versionId_object(s) + ListObjectVersions_checksum(s) // Multipart upload Versioning_Multipart_Upload_success(s) Versioning_Multipart_Upload_overwrite_an_object(s) @@ -1162,6 +1163,7 @@ func GetIntTests() IntTests { "ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers, "ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj, "ListObjectVersions_single_null_versionId_object": ListObjectVersions_single_null_versionId_object, + "ListObjectVersions_checksum": ListObjectVersions_checksum, "Versioning_Multipart_Upload_success": Versioning_Multipart_Upload_success, "Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object, "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 8fe52305..9e86a6d0 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -5281,7 +5281,7 @@ func ListObjectVersions_VD_success(s *S3Conf) error { return err } - if !compareVersions(res.Versions, versions) { + if !compareVersions(versions, res.Versions) { return fmt.Errorf("expected object versions output to be %v, instead got %v", versions, res.Versions) } return nil @@ -14041,6 +14041,7 @@ func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error { Size: &lgth, VersionId: &nullVersionId, StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: out.res.ChecksumType, }, }, versions...) @@ -14134,6 +14135,7 @@ func Versioning_CopyObject_success(s *S3Conf) error { Size: &srcObjLen, VersionId: out.VersionId, StorageClass: types.ObjectVersionStorageClassStandard, + ChecksumType: out.CopyObjectResult.ChecksumType, }, }, dstObjVersions...) @@ -15055,7 +15057,7 @@ func Versioning_DeleteObject_suspended(s *S3Conf) error { }, } - if !compareVersions(res.Versions, versions) { + if !compareVersions(versions, res.Versions) { return fmt.Errorf("expected the versions to be %v, instead got %v", versions, res.Versions) } if !compareDelMarkers(res.DeleteMarkers, delMarkers) { @@ -15298,7 +15300,7 @@ func ListObjectVersions_list_single_object_versions(s *S3Conf) error { return err } - if !compareVersions(out.Versions, versions) { + if !compareVersions(versions, out.Versions) { return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, out.Versions) } @@ -15335,7 +15337,7 @@ func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error { return err } - if !compareVersions(out.Versions, versions) { + if !compareVersions(versions, out.Versions) { return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, out.Versions) } @@ -15390,7 +15392,7 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { return fmt.Errorf("expected the NextVersionIdMarker to be %v, instead got %v", *versions[maxKeys].VersionId, *out.NextVersionIdMarker) } - if !compareVersions(out.Versions, versions[:maxKeys]) { + if !compareVersions(versions[:maxKeys], out.Versions) { return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", versions[:maxKeys], out.Versions) } @@ -15418,7 +15420,7 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error { return fmt.Errorf("expected the VersionIdMarker to be %v, instead got %v", *versions[maxKeys].VersionId, *out.VersionIdMarker) } - if !compareVersions(out.Versions, versions[maxKeys:]) { + if !compareVersions(versions[maxKeys:], out.Versions) { return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", versions[maxKeys:], out.Versions) } @@ -15463,7 +15465,7 @@ func ListObjectVersions_with_delete_markers(s *S3Conf) error { return err } - if !compareVersions(res.Versions, versions) { + if !compareVersions(versions, res.Versions) { return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, res.Versions) } if !compareDelMarkers(res.DeleteMarkers, delMarkers) { @@ -15535,7 +15537,7 @@ func ListObjectVersions_containing_null_versionId_obj(s *S3Conf) error { return err } - if !compareVersions(res.Versions, versions) { + if !compareVersions(versions, res.Versions) { return fmt.Errorf("expected the listed object versions to be %v, instead got %v", versions, res.Versions) } @@ -15600,7 +15602,7 @@ func ListObjectVersions_single_null_versionId_object(s *S3Conf) error { if !compareDelMarkers(resp.DeleteMarkers, delMarkers) { return fmt.Errorf("expected the delete markers list to be %v, instaed got %v", delMarkers, resp.DeleteMarkers) } - if !compareVersions(resp.Versions, versions) { + if !compareVersions(versions, resp.Versions) { return fmt.Errorf("expected the object versions list to be %v, instead got %v", versions, resp.Versions) } @@ -15608,6 +15610,36 @@ func ListObjectVersions_single_null_versionId_object(s *S3Conf) error { }) } +func ListObjectVersions_checksum(s *S3Conf) error { + testName := "ListObjectVersions_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + versions := []types.ObjectVersion{} + for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + vers, err := createObjVersions(s3client, bucket, fmt.Sprintf("obj-%v", i), 1, withChecksumAlgo(algo)) + if err != nil { + return err + } + + versions = append(versions, vers...) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareVersions(versions, res.Versions) { + return fmt.Errorf("expected the versions to be %+v, instead got %+v", versions, res.Versions) + } + + return nil + }, withVersioning(types.BucketVersioningStatusEnabled)) +} + func Versioning_Multipart_Upload_success(s *S3Conf) error { testName := "Versioning_Multipart_Upload_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -15761,7 +15793,7 @@ func Versioning_Multipart_Upload_overwrite_an_object(s *S3Conf) error { }, }, objVersions...) - if !compareVersions(resp.Versions, versions) { + if !compareVersions(versions, resp.Versions) { return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, resp.Versions) } diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 360ff920..c533e042 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -1112,7 +1112,22 @@ func pfxStrings(pfxs []types.CommonPrefix) []string { return pfxStrs } -func createObjVersions(client *s3.Client, bucket, object string, count int) ([]types.ObjectVersion, error) { +type versCfg struct { + checksumAlgorithm types.ChecksumAlgorithm +} + +type versOpt func(*versCfg) + +func withChecksumAlgo(algo types.ChecksumAlgorithm) versOpt { + return func(vc *versCfg) { vc.checksumAlgorithm = algo } +} + +func createObjVersions(client *s3.Client, bucket, object string, count int, opts ...versOpt) ([]types.ObjectVersion, error) { + cfg := new(versCfg) + for _, o := range opts { + o(cfg) + } + versions := []types.ObjectVersion{} for i := 0; i < count; i++ { rNumber, err := rand.Int(rand.Reader, big.NewInt(100000)) @@ -1130,15 +1145,40 @@ func createObjVersions(client *s3.Client, bucket, object string, count int) ([]t } isLatest := i == count-1 - - versions = append(versions, types.ObjectVersion{ + version := types.ObjectVersion{ ETag: r.res.ETag, IsLatest: &isLatest, Key: &object, Size: &dataLength, VersionId: r.res.VersionId, StorageClass: types.ObjectVersionStorageClassStandard, - }) + ChecksumType: r.res.ChecksumType, + } + + switch { + case r.res.ChecksumCRC32 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmCrc32, + } + case r.res.ChecksumCRC32C != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmCrc32c, + } + case r.res.ChecksumCRC64NVME != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmCrc64nvme, + } + case r.res.ChecksumSHA1 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmSha1, + } + case r.res.ChecksumSHA256 != nil: + version.ChecksumAlgorithm = []types.ChecksumAlgorithm{ + types.ChecksumAlgorithmSha256, + } + } + + versions = append(versions, version) } versions = reverseSlice(versions) @@ -1198,6 +1238,19 @@ func compareVersions(v1, v2 []types.ObjectVersion) bool { if version.StorageClass != v2[i].StorageClass { return false } + if version.ChecksumType != "" { + if version.ChecksumType != v2[i].ChecksumType { + return false + } + } + if len(version.ChecksumAlgorithm) != 0 { + if len(v2[i].ChecksumAlgorithm) == 0 { + return false + } + if version.ChecksumAlgorithm[0] != v2[i].ChecksumAlgorithm[0] { + return false + } + } } return true