Merge pull request #1090 from versity/feat/versioning-checksums

feat: Implements checksums for ListObjectVersions and CopyObject acti…

backend/posix/posix.go

@@ -915,14 +915,22 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
 					Key:          &path,
 				})
 			} else {
+				// Retreive checksum
+				checksum, err := p.retrieveChecksums(nil, bucket, path)
+				if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+					return nil, fmt.Errorf("get checksum: %w", err)
+				}
+
 				objects = append(objects, types.ObjectVersion{
-					ETag:         &etag,
-					Key:          &path,
-					LastModified: backend.GetTimePtr(fi.ModTime()),
-					Size:         &size,
-					VersionId:    &versionId,
-					IsLatest:     getBoolPtr(true),
-					StorageClass: types.ObjectVersionStorageClassStandard,
+					ETag:              &etag,
+					Key:               &path,
+					LastModified:      backend.GetTimePtr(fi.ModTime()),
+					Size:              &size,
+					VersionId:         &versionId,
+					IsLatest:          getBoolPtr(true),
+					StorageClass:      types.ObjectVersionStorageClassStandard,
+					ChecksumAlgorithm: []types.ChecksumAlgorithm{checksum.Algorithm},
+					ChecksumType:      checksum.Type,
 				})
 			}
 
@@ -998,6 +1006,12 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
 				// so this will just set etag to "" if its not already set
 				etag := string(etagBytes)
 				size := nf.Size()
+				// Retreive checksum
+				checksum, err := p.retrieveChecksums(nil, versionPath, nullVersionId)
+				if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+					return nil, fmt.Errorf("get checksum: %w", err)
+				}
+
 				nullVersionIdObj = &types.ObjectVersion{
 					ETag:         &etag,
 					Key:          &path,
@@ -1006,6 +1020,10 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
 					VersionId:    backend.GetPtrFromString("null"),
 					IsLatest:     getBoolPtr(false),
 					StorageClass: types.ObjectVersionStorageClassStandard,
+					ChecksumAlgorithm: []types.ChecksumAlgorithm{
+						checksum.Algorithm,
+					},
+					ChecksumType: checksum.Type,
 				}
 			}
 		}
@@ -1110,14 +1128,21 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
 					IsLatest:     getBoolPtr(false),
 				})
 			} else {
+				// Retreive checksum
+				checksum, err := p.retrieveChecksums(nil, versionPath, versionId)
+				if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
+					return nil, fmt.Errorf("get checksum: %w", err)
+				}
 				objects = append(objects, types.ObjectVersion{
-					ETag:         &etag,
-					Key:          &path,
-					LastModified: backend.GetTimePtr(f.ModTime()),
-					Size:         &size,
-					VersionId:    &versionId,
-					IsLatest:     getBoolPtr(false),
-					StorageClass: types.ObjectVersionStorageClassStandard,
+					ETag:              &etag,
+					Key:               &path,
+					LastModified:      backend.GetTimePtr(f.ModTime()),
+					Size:              &size,
+					VersionId:         &versionId,
+					IsLatest:          getBoolPtr(false),
+					StorageClass:      types.ObjectVersionStorageClassStandard,
+					ChecksumAlgorithm: []types.ChecksumAlgorithm{checksum.Algorithm},
+					ChecksumType:      checksum.Type,
 				})
 			}
 
@@ -3855,6 +3880,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 	var sha1 *string
 	var sha256 *string
 	var crc64nvme *string
+	var chType types.ChecksumType
 
 	dstObjdPath := filepath.Join(dstBucket, dstObject)
 	if dstObjdPath == objPath {
@@ -3882,6 +3908,8 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 			return nil, fmt.Errorf("get obj checksums: %w", err)
 		}
 
+		chType = checksums.Type
+
 		if input.ChecksumAlgorithm != "" {
 			// If a different checksum algorith is specified
 			// first caclculate and store the checksum
@@ -3923,6 +3951,10 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 					crc64nvme = &sum
 				}
 
+				// If a new checksum is calculated, the checksum type
+				// should be FULL_OBJECT
+				chType = types.ChecksumTypeFullObject
+
 				err = p.storeChecksums(f, dstBucket, dstObject, checksums)
 				if err != nil {
 					return nil, fmt.Errorf("store checksum: %w", err)
@@ -3970,6 +4002,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 		sha1 = res.ChecksumSHA1
 		sha256 = res.ChecksumSHA256
 		crc64nvme = res.ChecksumCRC64NVME
+		chType = res.ChecksumType
 	}
 
 	fi, err = os.Stat(dstObjdPath)
@@ -3986,6 +4019,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 			ChecksumSHA1:      sha1,
 			ChecksumSHA256:    sha256,
 			ChecksumCRC64NVME: crc64nvme,
+			ChecksumType:      chType,
 		},
 		VersionId:           version,
 		CopySourceVersionId: &srcVersionId,

tests/integration/group-tests.go

@@ -703,6 +703,7 @@ func TestVersioning(s *S3Conf) {
 	ListObjectVersions_with_delete_markers(s)
 	ListObjectVersions_containing_null_versionId_obj(s)
 	ListObjectVersions_single_null_versionId_object(s)
+	ListObjectVersions_checksum(s)
 	// Multipart upload
 	Versioning_Multipart_Upload_success(s)
 	Versioning_Multipart_Upload_overwrite_an_object(s)
@@ -1162,6 +1163,7 @@ func GetIntTests() IntTests {
 		"ListObjectVersions_with_delete_markers":                                  ListObjectVersions_with_delete_markers,
 		"ListObjectVersions_containing_null_versionId_obj":                        ListObjectVersions_containing_null_versionId_obj,
 		"ListObjectVersions_single_null_versionId_object":                         ListObjectVersions_single_null_versionId_object,
+		"ListObjectVersions_checksum":                                             ListObjectVersions_checksum,
 		"Versioning_Multipart_Upload_success":                                     Versioning_Multipart_Upload_success,
 		"Versioning_Multipart_Upload_overwrite_an_object":                         Versioning_Multipart_Upload_overwrite_an_object,
 		"Versioning_UploadPartCopy_non_existing_versionId":                        Versioning_UploadPartCopy_non_existing_versionId,

tests/integration/tests.go

@@ -5281,7 +5281,7 @@ func ListObjectVersions_VD_success(s *S3Conf) error {
 			return err
 		}
 
-		if !compareVersions(res.Versions, versions) {
+		if !compareVersions(versions, res.Versions) {
 			return fmt.Errorf("expected object versions output to be %v, instead got %v", versions, res.Versions)
 		}
 		return nil
@@ -14041,6 +14041,7 @@ func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error {
 				Size:         &lgth,
 				VersionId:    &nullVersionId,
 				StorageClass: types.ObjectVersionStorageClassStandard,
+				ChecksumType: out.res.ChecksumType,
 			},
 		}, versions...)
 
@@ -14134,6 +14135,7 @@ func Versioning_CopyObject_success(s *S3Conf) error {
 				Size:         &srcObjLen,
 				VersionId:    out.VersionId,
 				StorageClass: types.ObjectVersionStorageClassStandard,
+				ChecksumType: out.CopyObjectResult.ChecksumType,
 			},
 		}, dstObjVersions...)
 
@@ -15055,7 +15057,7 @@ func Versioning_DeleteObject_suspended(s *S3Conf) error {
 			},
 		}
 
-		if !compareVersions(res.Versions, versions) {
+		if !compareVersions(versions, res.Versions) {
 			return fmt.Errorf("expected the versions to be %v, instead got %v", versions, res.Versions)
 		}
 		if !compareDelMarkers(res.DeleteMarkers, delMarkers) {
@@ -15298,7 +15300,7 @@ func ListObjectVersions_list_single_object_versions(s *S3Conf) error {
 			return err
 		}
 
-		if !compareVersions(out.Versions, versions) {
+		if !compareVersions(versions, out.Versions) {
 			return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, out.Versions)
 		}
 
@@ -15335,7 +15337,7 @@ func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error {
 			return err
 		}
 
-		if !compareVersions(out.Versions, versions) {
+		if !compareVersions(versions, out.Versions) {
 			return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, out.Versions)
 		}
 
@@ -15390,7 +15392,7 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error {
 			return fmt.Errorf("expected the NextVersionIdMarker to be %v, instead got %v", *versions[maxKeys].VersionId, *out.NextVersionIdMarker)
 		}
 
-		if !compareVersions(out.Versions, versions[:maxKeys]) {
+		if !compareVersions(versions[:maxKeys], out.Versions) {
 			return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", versions[:maxKeys], out.Versions)
 		}
 
@@ -15418,7 +15420,7 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error {
 			return fmt.Errorf("expected the VersionIdMarker to be %v, instead got %v", *versions[maxKeys].VersionId, *out.VersionIdMarker)
 		}
 
-		if !compareVersions(out.Versions, versions[maxKeys:]) {
+		if !compareVersions(versions[maxKeys:], out.Versions) {
 			return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", versions[maxKeys:], out.Versions)
 		}
 
@@ -15463,7 +15465,7 @@ func ListObjectVersions_with_delete_markers(s *S3Conf) error {
 			return err
 		}
 
-		if !compareVersions(res.Versions, versions) {
+		if !compareVersions(versions, res.Versions) {
 			return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, res.Versions)
 		}
 		if !compareDelMarkers(res.DeleteMarkers, delMarkers) {
@@ -15535,7 +15537,7 @@ func ListObjectVersions_containing_null_versionId_obj(s *S3Conf) error {
 			return err
 		}
 
-		if !compareVersions(res.Versions, versions) {
+		if !compareVersions(versions, res.Versions) {
 			return fmt.Errorf("expected the listed object versions to be %v, instead got %v", versions, res.Versions)
 		}
 
@@ -15600,7 +15602,7 @@ func ListObjectVersions_single_null_versionId_object(s *S3Conf) error {
 		if !compareDelMarkers(resp.DeleteMarkers, delMarkers) {
 			return fmt.Errorf("expected the delete markers list to be %v, instaed got %v", delMarkers, resp.DeleteMarkers)
 		}
-		if !compareVersions(resp.Versions, versions) {
+		if !compareVersions(versions, resp.Versions) {
 			return fmt.Errorf("expected the object versions list to be %v, instead got %v", versions, resp.Versions)
 		}
 
@@ -15608,6 +15610,36 @@ func ListObjectVersions_single_null_versionId_object(s *S3Conf) error {
 	})
 }
 
+func ListObjectVersions_checksum(s *S3Conf) error {
+	testName := "ListObjectVersions_checksum"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		versions := []types.ObjectVersion{}
+		for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
+			vers, err := createObjVersions(s3client, bucket, fmt.Sprintf("obj-%v", i), 1, withChecksumAlgo(algo))
+			if err != nil {
+				return err
+			}
+
+			versions = append(versions, vers...)
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
+			Bucket: &bucket,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		if !compareVersions(versions, res.Versions) {
+			return fmt.Errorf("expected the versions to be %+v, instead got %+v", versions, res.Versions)
+		}
+
+		return nil
+	}, withVersioning(types.BucketVersioningStatusEnabled))
+}
+
 func Versioning_Multipart_Upload_success(s *S3Conf) error {
 	testName := "Versioning_Multipart_Upload_success"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -15761,7 +15793,7 @@ func Versioning_Multipart_Upload_overwrite_an_object(s *S3Conf) error {
 			},
 		}, objVersions...)
 
-		if !compareVersions(resp.Versions, versions) {
+		if !compareVersions(versions, resp.Versions) {
 			return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, resp.Versions)
 		}
 

tests/integration/utils.go

@@ -1112,7 +1112,22 @@ func pfxStrings(pfxs []types.CommonPrefix) []string {
 	return pfxStrs
 }
 
-func createObjVersions(client *s3.Client, bucket, object string, count int) ([]types.ObjectVersion, error) {
+type versCfg struct {
+	checksumAlgorithm types.ChecksumAlgorithm
+}
+
+type versOpt func(*versCfg)
+
+func withChecksumAlgo(algo types.ChecksumAlgorithm) versOpt {
+	return func(vc *versCfg) { vc.checksumAlgorithm = algo }
+}
+
+func createObjVersions(client *s3.Client, bucket, object string, count int, opts ...versOpt) ([]types.ObjectVersion, error) {
+	cfg := new(versCfg)
+	for _, o := range opts {
+		o(cfg)
+	}
+
 	versions := []types.ObjectVersion{}
 	for i := 0; i < count; i++ {
 		rNumber, err := rand.Int(rand.Reader, big.NewInt(100000))
@@ -1130,15 +1145,40 @@ func createObjVersions(client *s3.Client, bucket, object string, count int) ([]t
 		}
 
 		isLatest := i == count-1
-
-		versions = append(versions, types.ObjectVersion{
+		version := types.ObjectVersion{
 			ETag:         r.res.ETag,
 			IsLatest:     &isLatest,
 			Key:          &object,
 			Size:         &dataLength,
 			VersionId:    r.res.VersionId,
 			StorageClass: types.ObjectVersionStorageClassStandard,
-		})
+			ChecksumType: r.res.ChecksumType,
+		}
+
+		switch {
+		case r.res.ChecksumCRC32 != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmCrc32,
+			}
+		case r.res.ChecksumCRC32C != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmCrc32c,
+			}
+		case r.res.ChecksumCRC64NVME != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmCrc64nvme,
+			}
+		case r.res.ChecksumSHA1 != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmSha1,
+			}
+		case r.res.ChecksumSHA256 != nil:
+			version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
+				types.ChecksumAlgorithmSha256,
+			}
+		}
+
+		versions = append(versions, version)
 	}
 
 	versions = reverseSlice(versions)
@@ -1198,6 +1238,19 @@ func compareVersions(v1, v2 []types.ObjectVersion) bool {
 		if version.StorageClass != v2[i].StorageClass {
 			return false
 		}
+		if version.ChecksumType != "" {
+			if version.ChecksumType != v2[i].ChecksumType {
+				return false
+			}
+		}
+		if len(version.ChecksumAlgorithm) != 0 {
+			if len(v2[i].ChecksumAlgorithm) == 0 {
+				return false
+			}
+			if version.ChecksumAlgorithm[0] != v2[i].ChecksumAlgorithm[0] {
+				return false
+			}
+		}
 	}
 
 	return true