From d0884e7ce72b857a8470ed220afd214c7837d9d0 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Fri, 4 Sep 2026 19:45:12 +0800 Subject: [PATCH 1/5] return source size from uploader Signed-off-by: Lyndon-Li --- pkg/uploader/block/snapshot.go | 15 ++++++++------- pkg/uploader/kopia/snapshot.go | 5 +++-- pkg/uploader/provider/block.go | 18 +++++++++--------- pkg/uploader/provider/kopia.go | 18 +++++++++--------- pkg/uploader/provider/provider.go | 2 +- pkg/uploader/types.go | 3 ++- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/pkg/uploader/block/snapshot.go b/pkg/uploader/block/snapshot.go index 566ebad4f..720967ed2 100644 --- a/pkg/uploader/block/snapshot.go +++ b/pkg/uploader/block/snapshot.go @@ -79,11 +79,12 @@ func Backup(ctx context.Context, blkUp Uploader, repoWriter udmrepo.BackupRepo, return uploader.SnapshotInfo{}, false, errors.Wrapf(err, "error reset pos of block device %s", source) } - snapID, backupSize, err := snapshotSource(ctx, repoWriter, blkUp, sourceInfo, forceFull, parentSnapshot, cbtSource, cbtService, tags, uploaderCfg, log, "Block Uploader") + snapID, backupSize, snapshotSize, err := snapshotSource(ctx, repoWriter, blkUp, sourceInfo, forceFull, parentSnapshot, cbtSource, cbtService, tags, uploaderCfg, log, "Block Uploader") snapshotInfo := uploader.SnapshotInfo{ ID: snapID, - Size: sourceInfo.size, + SnapshotSize: snapshotSize, IncrementalSize: backupSize, + SourceSize: sourceInfo.size, } return snapshotInfo, false, err @@ -102,7 +103,7 @@ func snapshotSource( uploaderCfg map[string]string, log logrus.FieldLogger, description string, -) (string, int64, error) { +) (string, int64, int64, error) { log.Info("Start to snapshot...") snapshotStartTime := time.Now() @@ -118,7 +119,7 @@ func snapshotSource( snap, backupSize, err := u.Backup(source, parentBackup.parentObject, bitmap.Iterator(), uploaderCfg) if err != nil { - return "", 0, errors.Wrapf(err, "Failed to run uploader backup for si %v", source) + return "", 0, 0, errors.Wrapf(err, "Failed to run uploader backup for si %v", source) } if snap.Tags == nil { @@ -135,16 +136,16 @@ func snapshotSource( snapID, err := rep.SaveSnapshot(ctx, snap) if err != nil { - return "", 0, errors.Wrapf(err, "Failed to save snapshot %v", snap) + return "", 0, 0, errors.Wrapf(err, "Failed to save snapshot %v", snap) } if err = rep.Flush(ctx); err != nil { - return "", 0, errors.Wrapf(err, "Failed to flush repository") + return "", 0, 0, errors.Wrapf(err, "Failed to flush repository") } log.Infof("Created snapshot with root %v and ID %v in %v", snap.RootObject, snapID, time.Since(snapshotStartTime).Truncate(time.Second)) - return string(snapID), backupSize, nil + return string(snapID), backupSize, snap.TotalSize, nil } func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull bool, parentSnapshot string, volumeID string, realSource string, snapshotTags map[string]string, log logrus.FieldLogger) parentBackupInfo { diff --git a/pkg/uploader/kopia/snapshot.go b/pkg/uploader/kopia/snapshot.go index fae7a517c..85c92eac8 100644 --- a/pkg/uploader/kopia/snapshot.go +++ b/pkg/uploader/kopia/snapshot.go @@ -191,8 +191,9 @@ func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.Re snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, uploaderCfg, log, "Kopia Uploader") snapshotInfo := &uploader.SnapshotInfo{ - ID: snapID, - Size: snapshotSize, + ID: snapID, + SnapshotSize: snapshotSize, + SourceSize: snapshotSize, } return snapshotInfo, false, err diff --git a/pkg/uploader/provider/block.go b/pkg/uploader/provider/block.go index 3d51e275b..73c90a798 100644 --- a/pkg/uploader/provider/block.go +++ b/pkg/uploader/provider/block.go @@ -103,13 +103,13 @@ func (bp *blockProvider) RunBackup( cbtParam CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, - updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { + updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error) { if updater == nil { - return "", false, 0, 0, errors.New("backup progress updater is invalid") + return "", false, 0, 0, 0, errors.New("backup progress updater is invalid") } if path == "" { - return "", false, 0, 0, errors.New("path is empty") + return "", false, 0, 0, 0, errors.New("path is empty") } log := bp.log.WithFields(logrus.Fields{ @@ -140,23 +140,23 @@ func (bp *blockProvider) RunBackup( // equality check never matches and cancellation gets reported as a failure. if errors.Is(err, block.ErrCanceled) { log.Warn("Block backup is canceled") - return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, ErrorCanceled + return snapshotInfo.ID, false, snapshotInfo.SnapshotSize, snapshotInfo.IncrementalSize, snapshotInfo.SourceSize, ErrorCanceled } if err != nil { - return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, errors.Wrapf(err, "Failed to run block backup") + return snapshotInfo.ID, false, snapshotInfo.SnapshotSize, snapshotInfo.IncrementalSize, snapshotInfo.SourceSize, errors.Wrapf(err, "Failed to run block backup") } updater.UpdateProgress( &uploader.Progress{ - TotalBytes: snapshotInfo.Size, - BytesDone: snapshotInfo.Size, + TotalBytes: snapshotInfo.SnapshotSize, + BytesDone: snapshotInfo.SnapshotSize, }, ) - log.Infof("Block backup finished, snapshot ID %s, backup size %v, incremental size %v", snapshotInfo.ID, snapshotInfo.Size, snapshotInfo.IncrementalSize) + log.Infof("Block backup finished, snapshot ID %s, backup size %v, incremental size %v, source size %v", snapshotInfo.ID, snapshotInfo.SnapshotSize, snapshotInfo.IncrementalSize, snapshotInfo.SourceSize) - return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, nil + return snapshotInfo.ID, false, snapshotInfo.SnapshotSize, snapshotInfo.IncrementalSize, snapshotInfo.SourceSize, nil } func (bp *blockProvider) RunRestore( diff --git a/pkg/uploader/provider/kopia.go b/pkg/uploader/provider/kopia.go index 1cd1ecd45..ac35836f8 100644 --- a/pkg/uploader/provider/kopia.go +++ b/pkg/uploader/provider/kopia.go @@ -121,13 +121,13 @@ func (kp *kopiaProvider) RunBackup( volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater, -) (string, bool, int64, int64, error) { +) (string, bool, int64, int64, int64, error) { if updater == nil { - return "", false, 0, 0, errors.New("Need to initial backup progress updater first") + return "", false, 0, 0, 0, errors.New("Need to initial backup progress updater first") } if path == "" { - return "", false, 0, 0, errors.New("path is empty") + return "", false, 0, 0, 0, errors.New("path is empty") } log := kp.log.WithFields(logrus.Fields{ @@ -177,21 +177,21 @@ func (kp *kopiaProvider) RunBackup( if kpUploader.IsCanceled() { log.Warn("Kopia backup is canceled") - return snapshotID, false, 0, 0, ErrorCanceled + return snapshotID, false, 0, 0, 0, ErrorCanceled } - return snapshotID, false, 0, 0, errors.Wrapf(err, "Failed to run kopia backup") + return snapshotID, false, 0, 0, 0, errors.Wrapf(err, "Failed to run kopia backup") } // which ensure that the statistic data of TotalBytes equal to BytesDone when finished updater.UpdateProgress( &uploader.Progress{ - TotalBytes: snapshotInfo.Size, - BytesDone: snapshotInfo.Size, + TotalBytes: snapshotInfo.SnapshotSize, + BytesDone: snapshotInfo.SnapshotSize, }, ) - log.Debugf("Kopia backup finished, snapshot ID %s, backup size %d", snapshotInfo.ID, snapshotInfo.Size) - return snapshotInfo.ID, false, snapshotInfo.Size, progress.GetIncrementalSize(), nil + log.Debugf("Kopia backup finished, snapshot ID %s, backup size %d", snapshotInfo.ID, snapshotInfo.SnapshotSize) + return snapshotInfo.ID, false, snapshotInfo.SnapshotSize, progress.GetIncrementalSize(), snapshotInfo.SourceSize, nil } func (kp *kopiaProvider) GetPassword(param any) (string, error) { diff --git a/pkg/uploader/provider/provider.go b/pkg/uploader/provider/provider.go index 7f003989d..53a44a4fb 100644 --- a/pkg/uploader/provider/provider.go +++ b/pkg/uploader/provider/provider.go @@ -57,7 +57,7 @@ type Provider interface { cbtParam CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, - updater uploader.ProgressUpdater) (string, bool, int64, int64, error) + updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error) // RunRestore which will do restore for one specific volume with given snapshot id and return error // updater is used for updating backup progress which implement by third-party RunRestore( diff --git a/pkg/uploader/types.go b/pkg/uploader/types.go index 9c700193f..9efb62514 100644 --- a/pkg/uploader/types.go +++ b/pkg/uploader/types.go @@ -52,8 +52,9 @@ func ValidateUploaderType(t string) (string, error) { type SnapshotInfo struct { ID string - Size int64 + SnapshotSize int64 IncrementalSize int64 + SourceSize int64 } // Progress which defined two variables to record progress From 6c869218766f6b5daefba7f0638a224b4fdb5160 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Fri, 4 Sep 2026 19:49:12 +0800 Subject: [PATCH 2/5] du support source size Signed-off-by: Lyndon-Li --- pkg/apis/velero/v2alpha1/data_upload_types.go | 4 ++++ pkg/controller/data_upload_controller.go | 2 ++ pkg/datapath/data_path.go | 11 +++++++++-- pkg/datapath/types.go | 1 + 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pkg/apis/velero/v2alpha1/data_upload_types.go b/pkg/apis/velero/v2alpha1/data_upload_types.go index db4c8d3a8..bc7d895dc 100644 --- a/pkg/apis/velero/v2alpha1/data_upload_types.go +++ b/pkg/apis/velero/v2alpha1/data_upload_types.go @@ -177,6 +177,10 @@ type DataUploadStatus struct { // +optional IncrementalBytes *int64 `json:"incrementalBytes,omitempty"` + // SourceSize holds the total size of the source volume. + // +optional + SourceSize int64 `json:"sourceSize,omitempty"` + // Node is name of the node where the DataUpload is processed. // +optional Node string `json:"node,omitempty"` diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index ae50a7741..ed4cb2ae8 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -514,6 +514,8 @@ func (r *DataUploadReconciler) OnDataUploadCompleted(ctx context.Context, namesp du.Status.Phase = velerov2alpha1api.DataUploadPhaseCompleted du.Status.SnapshotID = result.Backup.SnapshotID du.Status.IncrementalBytes = result.Backup.IncrementalBytes + du.Status.SourceSize = result.Backup.SourceSize + du.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} if result.Backup.EmptySnapshot { du.Status.Message = "volume was empty so no data was upload" diff --git a/pkg/datapath/data_path.go b/pkg/datapath/data_path.go index 0513619be..0a84025bf 100644 --- a/pkg/datapath/data_path.go +++ b/pkg/datapath/data_path.go @@ -197,7 +197,7 @@ func (dp *generalDataPath) StartBackup(source AccessPoint, uploaderConfig map[st dp.wgDataPath.Done() }() - snapshotID, emptySnapshot, totalBytes, incrementalBytes, err := dp.uploaderProv.RunBackup( + snapshotID, emptySnapshot, totalBytes, incrementalBytes, sourceSize, err := dp.uploaderProv.RunBackup( dp.ctx, source.ByPath, backupParam.RealSource, @@ -226,7 +226,14 @@ func (dp *generalDataPath) StartBackup(source AccessPoint, uploaderConfig map[st } dp.callbacks.OnFailed(context.Background(), dp.namespace, dp.jobName, dataPathErr) } else { - dp.callbacks.OnCompleted(context.Background(), dp.namespace, dp.jobName, Result{Backup: BackupResult{snapshotID, emptySnapshot, source, totalBytes, ptr.To(incrementalBytes)}}) + dp.callbacks.OnCompleted(context.Background(), dp.namespace, dp.jobName, Result{Backup: BackupResult{ + SnapshotID: snapshotID, + EmptySnapshot: emptySnapshot, + Source: source, + TotalBytes: totalBytes, + IncrementalBytes: ptr.To(incrementalBytes), + SourceSize: sourceSize, + }}) } }() diff --git a/pkg/datapath/types.go b/pkg/datapath/types.go index bc7d4a9a9..7fbf2a957 100644 --- a/pkg/datapath/types.go +++ b/pkg/datapath/types.go @@ -39,6 +39,7 @@ type BackupResult struct { // while a current mover reporting a genuine zero still serializes the key and // unmarshals to a non-nil zero, distinguishing "measured zero" from "not measured". IncrementalBytes *int64 `json:"incrementalBytes,omitempty"` + SourceSize int64 `json:"sourceSize,omitempty"` } // RestoreResult represents the result of a restore From e1600caab35423f045621761369d932a9ab52443 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Fri, 4 Sep 2026 19:50:45 +0800 Subject: [PATCH 3/5] record source size to volume info Signed-off-by: Lyndon-Li --- internal/volume/volumes_information.go | 3 +++ pkg/backup/backup.go | 1 + 2 files changed, 4 insertions(+) diff --git a/internal/volume/volumes_information.go b/internal/volume/volumes_information.go index cec2922d9..50b614caf 100644 --- a/internal/volume/volumes_information.go +++ b/internal/volume/volumes_information.go @@ -181,6 +181,9 @@ type SnapshotDataMovementInfo struct { // incremental and must stay distinguishable from "unknown". IncrementalSize *int64 `json:"incrementalSize,omitempty"` + // The size of source volume, for backup only + SourceSize int64 `json:"sourceSize,omitempty"` + // The DataUpload's Status.Phase value Phase velerov2alpha1.DataUploadPhase } diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 038b85cc1..ce2addaec 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -1325,6 +1325,7 @@ func updateVolumeInfos( volumeInfos[index].SnapshotDataMovementInfo.RetainedSnapshot = dataUpload.Spec.CSISnapshot.VolumeSnapshot volumeInfos[index].SnapshotDataMovementInfo.Size = dataUpload.Status.Progress.TotalBytes volumeInfos[index].SnapshotDataMovementInfo.IncrementalSize = dataUpload.Status.IncrementalBytes + volumeInfos[index].SnapshotDataMovementInfo.SourceSize = dataUpload.Status.SourceSize volumeInfos[index].SnapshotDataMovementInfo.Phase = dataUpload.Status.Phase if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCompleted { From e0e600d715df4361d5a5c61842894e0ddb33e70c Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Wed, 9 Sep 2026 16:09:08 +0800 Subject: [PATCH 4/5] save source size for PVB Signed-off-by: Lyndon-Li --- internal/volume/volumes_information.go | 4 ++++ pkg/apis/velero/v1/pod_volume_backup_types.go | 4 ++++ pkg/controller/pod_volume_backup_controller.go | 1 + 3 files changed, 9 insertions(+) diff --git a/internal/volume/volumes_information.go b/internal/volume/volumes_information.go index cada725e2..c5af900cb 100644 --- a/internal/volume/volumes_information.go +++ b/internal/volume/volumes_information.go @@ -280,6 +280,9 @@ type PodVolumeBackupInfo struct { // the uploader reported no figure; a pointer to 0 means it transferred nothing. IncrementalSize *int64 `json:"incrementalSize,omitempty"` + // The size of source volume, for backup only + SourceSize int64 `json:"sourceSize,omitempty"` + // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. UploaderType string `json:"uploaderType"` @@ -342,6 +345,7 @@ func newPodVolumeInfoFromPVB(pvb *velerov1api.PodVolumeBackup) *PodVolumeBackupI SnapshotHandle: pvb.Status.SnapshotID, Size: pvb.Status.Progress.TotalBytes, IncrementalSize: pvb.Status.IncrementalBytes, + SourceSize: pvb.Status.SourceSize, UploaderType: pvb.Spec.UploaderType, VolumeName: pvb.Spec.Volume, PodName: pvb.Spec.Pod.Name, diff --git a/pkg/apis/velero/v1/pod_volume_backup_types.go b/pkg/apis/velero/v1/pod_volume_backup_types.go index c4b7f879c..559d784f5 100644 --- a/pkg/apis/velero/v1/pod_volume_backup_types.go +++ b/pkg/apis/velero/v1/pod_volume_backup_types.go @@ -132,6 +132,10 @@ type PodVolumeBackupStatus struct { // +optional IncrementalBytes *int64 `json:"incrementalBytes,omitempty"` + // SourceSize holds the total size of the source volume. + // +optional + SourceSize int64 `json:"sourceSize,omitempty"` + // AcceptedTimestamp records the time the pod volume backup is to be prepared. // The server's time is used for AcceptedTimestamp // +optional diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index c4e68ce33..80d0f2fff 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -551,6 +551,7 @@ func (r *PodVolumeBackupReconciler) OnDataPathCompleted(ctx context.Context, nam pvb.Status.SnapshotID = result.Backup.SnapshotID pvb.Status.CompletionTimestamp = &completionTime pvb.Status.IncrementalBytes = result.Backup.IncrementalBytes + pvb.Status.SourceSize = result.Backup.SourceSize if result.Backup.EmptySnapshot { pvb.Status.Message = "volume was empty so no snapshot was taken" } From daed42b1d882e4d1f237d56258101c9e99767f5f Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Wed, 9 Sep 2026 16:20:19 +0800 Subject: [PATCH 5/5] save source size to volume info for DU and PVB Signed-off-by: Lyndon-Li --- changelogs/unreleased/10506-Lyndon-Li | 1 + pkg/datamover/backup_micro_service_test.go | 2 +- pkg/datapath/data_path_test.go | 32 +++++++++-- pkg/podvolume/backup_micro_service_test.go | 2 +- pkg/uploader/block/snapshot_test.go | 63 ++++++++++++++-------- pkg/uploader/provider/block_test.go | 62 +++++++++++---------- pkg/uploader/provider/kopia_test.go | 2 +- pkg/uploader/provider/mocks/Provider.go | 24 +++++---- 8 files changed, 121 insertions(+), 67 deletions(-) create mode 100644 changelogs/unreleased/10506-Lyndon-Li diff --git a/changelogs/unreleased/10506-Lyndon-Li b/changelogs/unreleased/10506-Lyndon-Li new file mode 100644 index 000000000..b80faa461 --- /dev/null +++ b/changelogs/unreleased/10506-Lyndon-Li @@ -0,0 +1 @@ +Save source size to volume info for DU and PVB \ No newline at end of file diff --git a/pkg/datamover/backup_micro_service_test.go b/pkg/datamover/backup_micro_service_test.go index 69a4a1381..392781443 100644 --- a/pkg/datamover/backup_micro_service_test.go +++ b/pkg/datamover/backup_micro_service_test.go @@ -152,7 +152,7 @@ func TestOnDataUploadCompleted(t *testing.T) { { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), - expectedErr: "Failed to marshal backup result { false { } 0 }: fake-marshal-error", + expectedErr: "Failed to marshal backup result { false { } 0 0}: fake-marshal-error", }, { name: "succeed", diff --git a/pkg/datapath/data_path_test.go b/pkg/datapath/data_path_test.go index 495c949f9..8f8cf285f 100644 --- a/pkg/datapath/data_path_test.go +++ b/pkg/datapath/data_path_test.go @@ -71,7 +71,7 @@ func TestAsyncBackup(t *testing.T) { err: provider.ErrorCanceled, }, { - name: "async backup complete", + name: "async backup complete with totalBytes larger than sourceSize", callbacks: Callbacks{ OnFailed: nil, OnCancelled: nil, @@ -86,8 +86,32 @@ func TestAsyncBackup(t *testing.T) { SnapshotID: "fake-snapshot", EmptySnapshot: false, Source: AccessPoint{ByPath: "fake-path"}, - TotalBytes: 1000, - IncrementalBytes: ptr.To(int64(0)), + TotalBytes: 3000, + IncrementalBytes: ptr.To(int64(200)), + SourceSize: 2000, + }, + }, + path: "fake-path", + }, + { + name: "async backup complete with totalBytes equal to sourceSize", + callbacks: Callbacks{ + OnFailed: nil, + OnCancelled: nil, + OnCompleted: func(ctx context.Context, namespace string, job string, result Result) { + asyncResult = result + asyncErr = nil + finish <- struct{}{} + }, + }, + result: Result{ + Backup: BackupResult{ + SnapshotID: "fake-snapshot", + EmptySnapshot: false, + Source: AccessPoint{ByPath: "fake-path"}, + TotalBytes: 2000, + IncrementalBytes: ptr.To(int64(200)), + SourceSize: 2000, }, }, path: "fake-path", @@ -102,7 +126,7 @@ func TestAsyncBackup(t *testing.T) { if test.result.Backup.IncrementalBytes != nil { incrementalBytes = *test.result.Backup.IncrementalBytes } - mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, incrementalBytes, test.err) + mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, incrementalBytes, test.result.Backup.SourceSize, test.err) mockProvider.On("Close", mock.Anything).Return(nil) dp.uploaderProv = mockProvider dp.initialized = true diff --git a/pkg/podvolume/backup_micro_service_test.go b/pkg/podvolume/backup_micro_service_test.go index b83bafd8a..8eb5f7cf2 100644 --- a/pkg/podvolume/backup_micro_service_test.go +++ b/pkg/podvolume/backup_micro_service_test.go @@ -156,7 +156,7 @@ func TestOnDataPathCompleted(t *testing.T) { { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), - expectedErr: "Failed to marshal backup result { false { } 0 }: fake-marshal-error", + expectedErr: "Failed to marshal backup result { false { } 0 0}: fake-marshal-error", }, { name: "succeed", diff --git a/pkg/uploader/block/snapshot_test.go b/pkg/uploader/block/snapshot_test.go index 260f94b82..a64f4acae 100644 --- a/pkg/uploader/block/snapshot_test.go +++ b/pkg/uploader/block/snapshot_test.go @@ -106,14 +106,17 @@ func TestBackup(t *testing.T) { expectedErrStr: "Failed to run uploader backup", }, { - name: "success returns correct SnapshotInfo", + name: "success returns correct SnapshotInfo with snapshotSize larger than sourceSize", setupOpenDev: func(t *testing.T) *os.File { t.Helper() return tempFile(t, "test-block-data") }, setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil) + Return(udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root"}, + TotalSize: int64(2048), + }, int64(8), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil) repo.On("Flush", mock.Anything).Return(nil) }, @@ -121,24 +124,31 @@ func TestBackup(t *testing.T) { t.Helper() assert.Equal(t, "snap-001", info.ID) assert.Equal(t, int64(8), info.IncrementalSize) - assert.Positive(t, info.Size) + assert.Equal(t, int64(2048), info.SnapshotSize) + assert.Equal(t, int64(len("test-block-data")), info.SourceSize) }, }, { - name: "success with CBT", + name: "success with CBT and snapshotSize equal to sourceSize", setupOpenDev: func(t *testing.T) *os.File { t.Helper() return tempFile(t, "test-block-data") }, setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil) + Return(udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root"}, + TotalSize: int64(len("test-block-data")), + }, int64(8), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil) repo.On("Flush", mock.Anything).Return(nil) }, checkInfo: func(t *testing.T, info uploader.SnapshotInfo) { t.Helper() assert.Equal(t, "snap-001", info.ID) + assert.Equal(t, int64(8), info.IncrementalSize) + assert.Equal(t, int64(len("test-block-data")), info.SnapshotSize) + assert.Equal(t, int64(len("test-block-data")), info.SourceSize) }, }, } @@ -199,12 +209,13 @@ func TestSnapshotSource(t *testing.T) { baseSource := sourceInfo{realSource: "/test/vol", size: 1024} testCases := []struct { - name string - setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) - expectedErrStr string - expectedSnapID string - expectedSize int64 - cbtService func(t *testing.T) cbtservice.Service + name string + setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) + expectedErrStr string + expectedSnapID string + expectedSize int64 + expectedSnapshotSize int64 + cbtService func(t *testing.T) cbtservice.Service }{ { name: "uploader Backup error", @@ -241,18 +252,19 @@ func TestSnapshotSource(t *testing.T) { // In full mode, the iterator should cover the whole range if it's a full backup return iter != nil }), mock.Anything). - Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(512), nil) + Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}, TotalSize: 2048}, int64(512), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-success"), nil) repo.On("Flush", mock.Anything).Return(nil) }, - expectedSnapID: "snap-success", - expectedSize: 512, + expectedSnapID: "snap-success", + expectedSize: 512, + expectedSnapshotSize: 2048, }, { name: "tags from cbtSource and snapshotTags are merged onto snapshot", setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{}, int64(0), nil) + Return(udmrepo.Snapshot{TotalSize: 4096}, int64(256), nil) repo.On("SaveSnapshot", mock.Anything, mock.MatchedBy(func(snap udmrepo.Snapshot) bool { return snap.Tags[uploader.CBTChangeIDTag] == "cid-1" && snap.Tags[uploader.CBTVolumeIDTag] == "vid-1" && @@ -261,7 +273,9 @@ func TestSnapshotSource(t *testing.T) { })).Return(udmrepo.ID("snap-tags"), nil) repo.On("Flush", mock.Anything).Return(nil) }, - expectedSnapID: "snap-tags", + expectedSnapID: "snap-tags", + expectedSize: 256, + expectedSnapshotSize: 4096, }, { name: "success with cbtService getting allocated blocks", @@ -277,12 +291,13 @@ func TestSnapshotSource(t *testing.T) { }, setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(1024), nil) + Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}, TotalSize: 8192}, int64(1024), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-alloc"), nil) repo.On("Flush", mock.Anything).Return(nil) }, - expectedSnapID: "snap-cbt-alloc", - expectedSize: 1024, + expectedSnapID: "snap-cbt-alloc", + expectedSize: 1024, + expectedSnapshotSize: 8192, }, { name: "cbtService error falls back to full", @@ -296,12 +311,13 @@ func TestSnapshotSource(t *testing.T) { setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { // Should be called with parentObject as empty because of fallback blkup.On("Backup", mock.Anything, udmrepo.ID(""), mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{}, int64(2048), nil) + Return(udmrepo.Snapshot{TotalSize: 1024}, int64(1024), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-fallback"), nil) repo.On("Flush", mock.Anything).Return(nil) }, - expectedSnapID: "snap-cbt-fallback", - expectedSize: 2048, + expectedSnapID: "snap-cbt-fallback", + expectedSize: 1024, + expectedSnapshotSize: 1024, }, } @@ -321,7 +337,7 @@ func TestSnapshotSource(t *testing.T) { cbtSvc = tc.cbtService(t) } - snapID, size, err := snapshotSource( + snapID, size, snapshotSize, err := snapshotSource( ctx, mockRepo, mockBlkup, baseSource, true, "", @@ -337,6 +353,7 @@ func TestSnapshotSource(t *testing.T) { require.NoError(t, err) assert.Equal(t, tc.expectedSnapID, snapID) assert.Equal(t, tc.expectedSize, size) + assert.Equal(t, tc.expectedSnapshotSize, snapshotSize) } mockBlkup.AssertExpectations(t) diff --git a/pkg/uploader/provider/block_test.go b/pkg/uploader/provider/block_test.go index e38de4985..b0067e7f4 100644 --- a/pkg/uploader/provider/block_test.go +++ b/pkg/uploader/provider/block_test.go @@ -204,20 +204,21 @@ func TestBlockProviderRunBackup(t *testing.T) { const requestorType = "test-requestor" testCases := []struct { - name string - path string - realSource string - tags map[string]string - updater uploader.ProgressUpdater - mockBackupResult uploader.SnapshotInfo - mockBackupErr error - expectedID string - expectedSize int64 - expectedIncrSize int64 - expectError bool - expectedErrStr string - skipMock bool - checkCaptures func(*testing.T, string, map[string]string) + name string + path string + realSource string + tags map[string]string + updater uploader.ProgressUpdater + mockBackupResult uploader.SnapshotInfo + mockBackupErr error + expectedID string + expectedSize int64 + expectedIncrSize int64 + expectedSourceSize int64 + expectError bool + expectedErrStr string + skipMock bool + checkCaptures func(*testing.T, string, map[string]string) }{ { name: "nil updater returns error", @@ -241,12 +242,14 @@ func TestBlockProviderRunBackup(t *testing.T) { updater: &blockMockProgressUpdater{}, mockBackupResult: uploader.SnapshotInfo{ ID: "snap-001", - Size: 1024, + SnapshotSize: 2048, IncrementalSize: 512, + SourceSize: 1024, }, - expectedID: "snap-001", - expectedSize: 1024, - expectedIncrSize: 512, + expectedID: "snap-001", + expectedSize: 2048, + expectedIncrSize: 512, + expectedSourceSize: 1024, }, { name: "canceled backup returns ErrorCanceled with partial snapshot info", @@ -254,15 +257,17 @@ func TestBlockProviderRunBackup(t *testing.T) { updater: &FakeBackupProgressUpdater{}, mockBackupResult: uploader.SnapshotInfo{ ID: "snap-canceled", - Size: 2048, + SnapshotSize: 2048, IncrementalSize: 1024, + SourceSize: 1024, }, - mockBackupErr: block.ErrCanceled, - expectedID: "snap-canceled", - expectedSize: 2048, - expectedIncrSize: 1024, - expectError: true, - expectedErrStr: "uploader is canceled", + mockBackupErr: block.ErrCanceled, + expectedID: "snap-canceled", + expectedSize: 2048, + expectedIncrSize: 1024, + expectedSourceSize: 1024, + expectError: true, + expectedErrStr: "uploader is canceled", }, { name: "generic backup error is wrapped", @@ -332,7 +337,7 @@ func TestBlockProviderRunBackup(t *testing.T) { log: logrus.New(), } - snapshotID, isEmpty, size, incrSize, err := bp.RunBackup( + snapshotID, isEmpty, size, incrSize, sourceSize, err := bp.RunBackup( t.Context(), tc.path, tc.realSource, @@ -348,6 +353,7 @@ func TestBlockProviderRunBackup(t *testing.T) { assert.Equal(t, tc.expectedID, snapshotID) assert.Equal(t, tc.expectedSize, size) assert.Equal(t, tc.expectedIncrSize, incrSize) + assert.Equal(t, tc.expectedSourceSize, sourceSize) if tc.expectError { require.Error(t, err) @@ -386,7 +392,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) { orig := blockBackupFunc defer func() { blockBackupFunc = orig }() blockBackupFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ cbtservice.SourceInfo, _ bool, _ string, _ cbtservice.Service, _ map[string]string, _ map[string]string, _ logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) { - return uploader.SnapshotInfo{ID: "snap-cancel", Size: 2048, IncrementalSize: 1024}, false, + return uploader.SnapshotInfo{ID: "snap-cancel", SnapshotSize: 2048, IncrementalSize: 1024, SourceSize: 1024}, false, errors.Wrapf( errors.Wrapf(block.ErrCanceled, "error backing up bdev %s", "ns/pvc"), "Failed to run uploader backup for si %v", "si") @@ -398,7 +404,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) { log: logrus.New(), } - _, _, _, _, err := bp.RunBackup( + _, _, _, _, _, err := bp.RunBackup( t.Context(), "/dev/sda", "ns/pvc", map[string]string{}, false, "", CBTParam{}, uploader.PersistentVolumeBlock, map[string]string{}, &FakeBackupProgressUpdater{}, diff --git a/pkg/uploader/provider/kopia_test.go b/pkg/uploader/provider/kopia_test.go index ca1cf8f5a..b0f8a9fd4 100644 --- a/pkg/uploader/provider/kopia_test.go +++ b/pkg/uploader/provider/kopia_test.go @@ -106,7 +106,7 @@ func TestRunBackup(t *testing.T) { tc.volMode = uploader.PersistentVolumeFilesystem } kopiaBackupFunc = tc.hookBackupFunc - _, _, _, _, err := kp.RunBackup(t.Context(), "var", "", nil, false, "", CBTParam{}, tc.volMode, map[string]string{}, &updater) + _, _, _, _, _, err := kp.RunBackup(t.Context(), "var", "", nil, false, "", CBTParam{}, tc.volMode, map[string]string{}, &updater) if tc.notError { assert.NoError(t, err) } else { diff --git a/pkg/uploader/provider/mocks/Provider.go b/pkg/uploader/provider/mocks/Provider.go index 7fad73e59..f2b8d764a 100644 --- a/pkg/uploader/provider/mocks/Provider.go +++ b/pkg/uploader/provider/mocks/Provider.go @@ -91,7 +91,7 @@ func (_c *Provider_Close_Call) RunAndReturn(run func(ctx context.Context) error) } // RunBackup provides a mock function for the type Provider -func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { +func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error) { ret := _mock.Called(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater) if len(ret) == 0 { @@ -102,8 +102,9 @@ func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource st var r1 bool var r2 int64 var r3 int64 - var r4 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, int64, int64, error)); ok { + var r4 int64 + var r5 error + if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, int64, int64, int64, error)); ok { return returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater) } if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) string); ok { @@ -126,12 +127,17 @@ func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource st } else { r3 = ret.Get(3).(int64) } - if returnFunc, ok := ret.Get(4).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok { + if returnFunc, ok := ret.Get(4).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok { r4 = returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater) } else { - r4 = ret.Error(4) + r4 = ret.Get(4).(int64) } - return r0, r1, r2, r3, r4 + if returnFunc, ok := ret.Get(5).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok { + r5 = returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater) + } else { + r5 = ret.Error(5) + } + return r0, r1, r2, r3, r4, r5 } // Provider_RunBackup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunBackup' @@ -212,12 +218,12 @@ func (_c *Provider_RunBackup_Call) Run(run func(ctx context.Context, path string return _c } -func (_c *Provider_RunBackup_Call) Return(s string, b bool, n int64, n1 int64, err error) *Provider_RunBackup_Call { - _c.Call.Return(s, b, n, n1, err) +func (_c *Provider_RunBackup_Call) Return(_a0 string, _a1 bool, _a2 int64, _a3 int64, _a4 int64, _a5 error) *Provider_RunBackup_Call { + _c.Call.Return(_a0, _a1, _a2, _a3, _a4, _a5) return _c } -func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error)) *Provider_RunBackup_Call { +func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error)) *Provider_RunBackup_Call { _c.Call.Return(run) return _c }