From daed42b1d882e4d1f237d56258101c9e99767f5f Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Wed, 9 Sep 2026 16:20:19 +0800 Subject: [PATCH] save source size to volume info for DU and PVB Signed-off-by: Lyndon-Li --- changelogs/unreleased/10506-Lyndon-Li | 1 + pkg/datamover/backup_micro_service_test.go | 2 +- pkg/datapath/data_path_test.go | 32 +++++++++-- pkg/podvolume/backup_micro_service_test.go | 2 +- pkg/uploader/block/snapshot_test.go | 63 ++++++++++++++-------- pkg/uploader/provider/block_test.go | 62 +++++++++++---------- pkg/uploader/provider/kopia_test.go | 2 +- pkg/uploader/provider/mocks/Provider.go | 24 +++++---- 8 files changed, 121 insertions(+), 67 deletions(-) create mode 100644 changelogs/unreleased/10506-Lyndon-Li diff --git a/changelogs/unreleased/10506-Lyndon-Li b/changelogs/unreleased/10506-Lyndon-Li new file mode 100644 index 000000000..b80faa461 --- /dev/null +++ b/changelogs/unreleased/10506-Lyndon-Li @@ -0,0 +1 @@ +Save source size to volume info for DU and PVB \ No newline at end of file diff --git a/pkg/datamover/backup_micro_service_test.go b/pkg/datamover/backup_micro_service_test.go index 69a4a1381..392781443 100644 --- a/pkg/datamover/backup_micro_service_test.go +++ b/pkg/datamover/backup_micro_service_test.go @@ -152,7 +152,7 @@ func TestOnDataUploadCompleted(t *testing.T) { { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), - expectedErr: "Failed to marshal backup result { false { } 0 }: fake-marshal-error", + expectedErr: "Failed to marshal backup result { false { } 0 0}: fake-marshal-error", }, { name: "succeed", diff --git a/pkg/datapath/data_path_test.go b/pkg/datapath/data_path_test.go index 495c949f9..8f8cf285f 100644 --- a/pkg/datapath/data_path_test.go +++ b/pkg/datapath/data_path_test.go @@ -71,7 +71,7 @@ func TestAsyncBackup(t *testing.T) { err: provider.ErrorCanceled, }, { - name: "async backup complete", + name: "async backup complete with totalBytes larger than sourceSize", callbacks: Callbacks{ OnFailed: nil, OnCancelled: nil, @@ -86,8 +86,32 @@ func TestAsyncBackup(t *testing.T) { SnapshotID: "fake-snapshot", EmptySnapshot: false, Source: AccessPoint{ByPath: "fake-path"}, - TotalBytes: 1000, - IncrementalBytes: ptr.To(int64(0)), + TotalBytes: 3000, + IncrementalBytes: ptr.To(int64(200)), + SourceSize: 2000, + }, + }, + path: "fake-path", + }, + { + name: "async backup complete with totalBytes equal to sourceSize", + callbacks: Callbacks{ + OnFailed: nil, + OnCancelled: nil, + OnCompleted: func(ctx context.Context, namespace string, job string, result Result) { + asyncResult = result + asyncErr = nil + finish <- struct{}{} + }, + }, + result: Result{ + Backup: BackupResult{ + SnapshotID: "fake-snapshot", + EmptySnapshot: false, + Source: AccessPoint{ByPath: "fake-path"}, + TotalBytes: 2000, + IncrementalBytes: ptr.To(int64(200)), + SourceSize: 2000, }, }, path: "fake-path", @@ -102,7 +126,7 @@ func TestAsyncBackup(t *testing.T) { if test.result.Backup.IncrementalBytes != nil { incrementalBytes = *test.result.Backup.IncrementalBytes } - mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, incrementalBytes, test.err) + mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, incrementalBytes, test.result.Backup.SourceSize, test.err) mockProvider.On("Close", mock.Anything).Return(nil) dp.uploaderProv = mockProvider dp.initialized = true diff --git a/pkg/podvolume/backup_micro_service_test.go b/pkg/podvolume/backup_micro_service_test.go index b83bafd8a..8eb5f7cf2 100644 --- a/pkg/podvolume/backup_micro_service_test.go +++ b/pkg/podvolume/backup_micro_service_test.go @@ -156,7 +156,7 @@ func TestOnDataPathCompleted(t *testing.T) { { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), - expectedErr: "Failed to marshal backup result { false { } 0 }: fake-marshal-error", + expectedErr: "Failed to marshal backup result { false { } 0 0}: fake-marshal-error", }, { name: "succeed", diff --git a/pkg/uploader/block/snapshot_test.go b/pkg/uploader/block/snapshot_test.go index 260f94b82..a64f4acae 100644 --- a/pkg/uploader/block/snapshot_test.go +++ b/pkg/uploader/block/snapshot_test.go @@ -106,14 +106,17 @@ func TestBackup(t *testing.T) { expectedErrStr: "Failed to run uploader backup", }, { - name: "success returns correct SnapshotInfo", + name: "success returns correct SnapshotInfo with snapshotSize larger than sourceSize", setupOpenDev: func(t *testing.T) *os.File { t.Helper() return tempFile(t, "test-block-data") }, setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil) + Return(udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root"}, + TotalSize: int64(2048), + }, int64(8), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil) repo.On("Flush", mock.Anything).Return(nil) }, @@ -121,24 +124,31 @@ func TestBackup(t *testing.T) { t.Helper() assert.Equal(t, "snap-001", info.ID) assert.Equal(t, int64(8), info.IncrementalSize) - assert.Positive(t, info.Size) + assert.Equal(t, int64(2048), info.SnapshotSize) + assert.Equal(t, int64(len("test-block-data")), info.SourceSize) }, }, { - name: "success with CBT", + name: "success with CBT and snapshotSize equal to sourceSize", setupOpenDev: func(t *testing.T) *os.File { t.Helper() return tempFile(t, "test-block-data") }, setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil) + Return(udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root"}, + TotalSize: int64(len("test-block-data")), + }, int64(8), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil) repo.On("Flush", mock.Anything).Return(nil) }, checkInfo: func(t *testing.T, info uploader.SnapshotInfo) { t.Helper() assert.Equal(t, "snap-001", info.ID) + assert.Equal(t, int64(8), info.IncrementalSize) + assert.Equal(t, int64(len("test-block-data")), info.SnapshotSize) + assert.Equal(t, int64(len("test-block-data")), info.SourceSize) }, }, } @@ -199,12 +209,13 @@ func TestSnapshotSource(t *testing.T) { baseSource := sourceInfo{realSource: "/test/vol", size: 1024} testCases := []struct { - name string - setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) - expectedErrStr string - expectedSnapID string - expectedSize int64 - cbtService func(t *testing.T) cbtservice.Service + name string + setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) + expectedErrStr string + expectedSnapID string + expectedSize int64 + expectedSnapshotSize int64 + cbtService func(t *testing.T) cbtservice.Service }{ { name: "uploader Backup error", @@ -241,18 +252,19 @@ func TestSnapshotSource(t *testing.T) { // In full mode, the iterator should cover the whole range if it's a full backup return iter != nil }), mock.Anything). - Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(512), nil) + Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}, TotalSize: 2048}, int64(512), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-success"), nil) repo.On("Flush", mock.Anything).Return(nil) }, - expectedSnapID: "snap-success", - expectedSize: 512, + expectedSnapID: "snap-success", + expectedSize: 512, + expectedSnapshotSize: 2048, }, { name: "tags from cbtSource and snapshotTags are merged onto snapshot", setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{}, int64(0), nil) + Return(udmrepo.Snapshot{TotalSize: 4096}, int64(256), nil) repo.On("SaveSnapshot", mock.Anything, mock.MatchedBy(func(snap udmrepo.Snapshot) bool { return snap.Tags[uploader.CBTChangeIDTag] == "cid-1" && snap.Tags[uploader.CBTVolumeIDTag] == "vid-1" && @@ -261,7 +273,9 @@ func TestSnapshotSource(t *testing.T) { })).Return(udmrepo.ID("snap-tags"), nil) repo.On("Flush", mock.Anything).Return(nil) }, - expectedSnapID: "snap-tags", + expectedSnapID: "snap-tags", + expectedSize: 256, + expectedSnapshotSize: 4096, }, { name: "success with cbtService getting allocated blocks", @@ -277,12 +291,13 @@ func TestSnapshotSource(t *testing.T) { }, setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(1024), nil) + Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}, TotalSize: 8192}, int64(1024), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-alloc"), nil) repo.On("Flush", mock.Anything).Return(nil) }, - expectedSnapID: "snap-cbt-alloc", - expectedSize: 1024, + expectedSnapID: "snap-cbt-alloc", + expectedSize: 1024, + expectedSnapshotSize: 8192, }, { name: "cbtService error falls back to full", @@ -296,12 +311,13 @@ func TestSnapshotSource(t *testing.T) { setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { // Should be called with parentObject as empty because of fallback blkup.On("Backup", mock.Anything, udmrepo.ID(""), mock.Anything, mock.Anything). - Return(udmrepo.Snapshot{}, int64(2048), nil) + Return(udmrepo.Snapshot{TotalSize: 1024}, int64(1024), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-fallback"), nil) repo.On("Flush", mock.Anything).Return(nil) }, - expectedSnapID: "snap-cbt-fallback", - expectedSize: 2048, + expectedSnapID: "snap-cbt-fallback", + expectedSize: 1024, + expectedSnapshotSize: 1024, }, } @@ -321,7 +337,7 @@ func TestSnapshotSource(t *testing.T) { cbtSvc = tc.cbtService(t) } - snapID, size, err := snapshotSource( + snapID, size, snapshotSize, err := snapshotSource( ctx, mockRepo, mockBlkup, baseSource, true, "", @@ -337,6 +353,7 @@ func TestSnapshotSource(t *testing.T) { require.NoError(t, err) assert.Equal(t, tc.expectedSnapID, snapID) assert.Equal(t, tc.expectedSize, size) + assert.Equal(t, tc.expectedSnapshotSize, snapshotSize) } mockBlkup.AssertExpectations(t) diff --git a/pkg/uploader/provider/block_test.go b/pkg/uploader/provider/block_test.go index e38de4985..b0067e7f4 100644 --- a/pkg/uploader/provider/block_test.go +++ b/pkg/uploader/provider/block_test.go @@ -204,20 +204,21 @@ func TestBlockProviderRunBackup(t *testing.T) { const requestorType = "test-requestor" testCases := []struct { - name string - path string - realSource string - tags map[string]string - updater uploader.ProgressUpdater - mockBackupResult uploader.SnapshotInfo - mockBackupErr error - expectedID string - expectedSize int64 - expectedIncrSize int64 - expectError bool - expectedErrStr string - skipMock bool - checkCaptures func(*testing.T, string, map[string]string) + name string + path string + realSource string + tags map[string]string + updater uploader.ProgressUpdater + mockBackupResult uploader.SnapshotInfo + mockBackupErr error + expectedID string + expectedSize int64 + expectedIncrSize int64 + expectedSourceSize int64 + expectError bool + expectedErrStr string + skipMock bool + checkCaptures func(*testing.T, string, map[string]string) }{ { name: "nil updater returns error", @@ -241,12 +242,14 @@ func TestBlockProviderRunBackup(t *testing.T) { updater: &blockMockProgressUpdater{}, mockBackupResult: uploader.SnapshotInfo{ ID: "snap-001", - Size: 1024, + SnapshotSize: 2048, IncrementalSize: 512, + SourceSize: 1024, }, - expectedID: "snap-001", - expectedSize: 1024, - expectedIncrSize: 512, + expectedID: "snap-001", + expectedSize: 2048, + expectedIncrSize: 512, + expectedSourceSize: 1024, }, { name: "canceled backup returns ErrorCanceled with partial snapshot info", @@ -254,15 +257,17 @@ func TestBlockProviderRunBackup(t *testing.T) { updater: &FakeBackupProgressUpdater{}, mockBackupResult: uploader.SnapshotInfo{ ID: "snap-canceled", - Size: 2048, + SnapshotSize: 2048, IncrementalSize: 1024, + SourceSize: 1024, }, - mockBackupErr: block.ErrCanceled, - expectedID: "snap-canceled", - expectedSize: 2048, - expectedIncrSize: 1024, - expectError: true, - expectedErrStr: "uploader is canceled", + mockBackupErr: block.ErrCanceled, + expectedID: "snap-canceled", + expectedSize: 2048, + expectedIncrSize: 1024, + expectedSourceSize: 1024, + expectError: true, + expectedErrStr: "uploader is canceled", }, { name: "generic backup error is wrapped", @@ -332,7 +337,7 @@ func TestBlockProviderRunBackup(t *testing.T) { log: logrus.New(), } - snapshotID, isEmpty, size, incrSize, err := bp.RunBackup( + snapshotID, isEmpty, size, incrSize, sourceSize, err := bp.RunBackup( t.Context(), tc.path, tc.realSource, @@ -348,6 +353,7 @@ func TestBlockProviderRunBackup(t *testing.T) { assert.Equal(t, tc.expectedID, snapshotID) assert.Equal(t, tc.expectedSize, size) assert.Equal(t, tc.expectedIncrSize, incrSize) + assert.Equal(t, tc.expectedSourceSize, sourceSize) if tc.expectError { require.Error(t, err) @@ -386,7 +392,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) { orig := blockBackupFunc defer func() { blockBackupFunc = orig }() blockBackupFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ cbtservice.SourceInfo, _ bool, _ string, _ cbtservice.Service, _ map[string]string, _ map[string]string, _ logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) { - return uploader.SnapshotInfo{ID: "snap-cancel", Size: 2048, IncrementalSize: 1024}, false, + return uploader.SnapshotInfo{ID: "snap-cancel", SnapshotSize: 2048, IncrementalSize: 1024, SourceSize: 1024}, false, errors.Wrapf( errors.Wrapf(block.ErrCanceled, "error backing up bdev %s", "ns/pvc"), "Failed to run uploader backup for si %v", "si") @@ -398,7 +404,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) { log: logrus.New(), } - _, _, _, _, err := bp.RunBackup( + _, _, _, _, _, err := bp.RunBackup( t.Context(), "/dev/sda", "ns/pvc", map[string]string{}, false, "", CBTParam{}, uploader.PersistentVolumeBlock, map[string]string{}, &FakeBackupProgressUpdater{}, diff --git a/pkg/uploader/provider/kopia_test.go b/pkg/uploader/provider/kopia_test.go index ca1cf8f5a..b0f8a9fd4 100644 --- a/pkg/uploader/provider/kopia_test.go +++ b/pkg/uploader/provider/kopia_test.go @@ -106,7 +106,7 @@ func TestRunBackup(t *testing.T) { tc.volMode = uploader.PersistentVolumeFilesystem } kopiaBackupFunc = tc.hookBackupFunc - _, _, _, _, err := kp.RunBackup(t.Context(), "var", "", nil, false, "", CBTParam{}, tc.volMode, map[string]string{}, &updater) + _, _, _, _, _, err := kp.RunBackup(t.Context(), "var", "", nil, false, "", CBTParam{}, tc.volMode, map[string]string{}, &updater) if tc.notError { assert.NoError(t, err) } else { diff --git a/pkg/uploader/provider/mocks/Provider.go b/pkg/uploader/provider/mocks/Provider.go index 7fad73e59..f2b8d764a 100644 --- a/pkg/uploader/provider/mocks/Provider.go +++ b/pkg/uploader/provider/mocks/Provider.go @@ -91,7 +91,7 @@ func (_c *Provider_Close_Call) RunAndReturn(run func(ctx context.Context) error) } // RunBackup provides a mock function for the type Provider -func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { +func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error) { ret := _mock.Called(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater) if len(ret) == 0 { @@ -102,8 +102,9 @@ func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource st var r1 bool var r2 int64 var r3 int64 - var r4 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, int64, int64, error)); ok { + var r4 int64 + var r5 error + if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, int64, int64, int64, error)); ok { return returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater) } if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) string); ok { @@ -126,12 +127,17 @@ func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource st } else { r3 = ret.Get(3).(int64) } - if returnFunc, ok := ret.Get(4).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok { + if returnFunc, ok := ret.Get(4).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok { r4 = returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater) } else { - r4 = ret.Error(4) + r4 = ret.Get(4).(int64) } - return r0, r1, r2, r3, r4 + if returnFunc, ok := ret.Get(5).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok { + r5 = returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater) + } else { + r5 = ret.Error(5) + } + return r0, r1, r2, r3, r4, r5 } // Provider_RunBackup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunBackup' @@ -212,12 +218,12 @@ func (_c *Provider_RunBackup_Call) Run(run func(ctx context.Context, path string return _c } -func (_c *Provider_RunBackup_Call) Return(s string, b bool, n int64, n1 int64, err error) *Provider_RunBackup_Call { - _c.Call.Return(s, b, n, n1, err) +func (_c *Provider_RunBackup_Call) Return(_a0 string, _a1 bool, _a2 int64, _a3 int64, _a4 int64, _a5 error) *Provider_RunBackup_Call { + _c.Call.Return(_a0, _a1, _a2, _a3, _a4, _a5) return _c } -func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error)) *Provider_RunBackup_Call { +func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error)) *Provider_RunBackup_Call { _c.Call.Return(run) return _c }