mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-13 11:34:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

save source size to volume info for DU and PVB

Browse Source 
Signed-off-by: Lyndon-Li <lyonghui@vmware.com>
This commit is contained in:
Lyndon-Li
2026-09-09 17:34:17 +08:00
parent e0e600d715
commit daed42b1d8
8 changed files with 121 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changelogs/unreleased/10506-Lyndon-Li
+1
View File
@@ -0,0 +1 @@
Save source size to volume info for DU and PVB
pkg/datamover/backup_micro_service_test.go
+1 -1
View File
@@ -152,7 +152,7 @@ func TestOnDataUploadCompleted(t *testing.T) {
{
name: "marshal fail",
marshalErr: errors.New("fake-marshal-error"),
expectedErr: "Failed to marshal backup result { false { } 0 <nil>}: fake-marshal-error",
expectedErr: "Failed to marshal backup result { false { } 0 <nil> 0}: fake-marshal-error",
},
{
name: "succeed",
pkg/datapath/data_path_test.go
+28 -4
View File
@@ -71,7 +71,7 @@ func TestAsyncBackup(t *testing.T) {
err: provider.ErrorCanceled,
},
{
name: "async backup complete",
name: "async backup complete with totalBytes larger than sourceSize",
callbacks: Callbacks{
OnFailed: nil,
OnCancelled: nil,
@@ -86,8 +86,32 @@ func TestAsyncBackup(t *testing.T) {
SnapshotID: "fake-snapshot",
EmptySnapshot: false,
Source: AccessPoint{ByPath: "fake-path"},
TotalBytes: 1000,
IncrementalBytes: ptr.To(int64(0)),
TotalBytes: 3000,
IncrementalBytes: ptr.To(int64(200)),
SourceSize: 2000,
},
},
path: "fake-path",
},
{
name: "async backup complete with totalBytes equal to sourceSize",
callbacks: Callbacks{
OnFailed: nil,
OnCancelled: nil,
OnCompleted: func(ctx context.Context, namespace string, job string, result Result) {
asyncResult = result
asyncErr = nil
finish <- struct{}{}
},
},
result: Result{
Backup: BackupResult{
SnapshotID: "fake-snapshot",
EmptySnapshot: false,
Source: AccessPoint{ByPath: "fake-path"},
TotalBytes: 2000,
IncrementalBytes: ptr.To(int64(200)),
SourceSize: 2000,
},
},
path: "fake-path",
@@ -102,7 +126,7 @@ func TestAsyncBackup(t *testing.T) {
if test.result.Backup.IncrementalBytes != nil {
incrementalBytes = *test.result.Backup.IncrementalBytes
}
mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, incrementalBytes, test.err)
mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, incrementalBytes, test.result.Backup.SourceSize, test.err)
mockProvider.On("Close", mock.Anything).Return(nil)
dp.uploaderProv = mockProvider
dp.initialized = true
pkg/podvolume/backup_micro_service_test.go
+1 -1
View File
@@ -156,7 +156,7 @@ func TestOnDataPathCompleted(t *testing.T) {
{
name: "marshal fail",
marshalErr: errors.New("fake-marshal-error"),
expectedErr: "Failed to marshal backup result { false { } 0 <nil>}: fake-marshal-error",
expectedErr: "Failed to marshal backup result { false { } 0 <nil> 0}: fake-marshal-error",
},
{
name: "succeed",
pkg/uploader/block/snapshot_test.go
+40 -23
View File
@@ -106,14 +106,17 @@ func TestBackup(t *testing.T) {
expectedErrStr: "Failed to run uploader backup",
},
{
name: "success returns correct SnapshotInfo",
name: "success returns correct SnapshotInfo with snapshotSize larger than sourceSize",
setupOpenDev: func(t *testing.T) *os.File {
t.Helper()
return tempFile(t, "test-block-data")
},
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil)
Return(udmrepo.Snapshot{
RootObject: udmrepo.ObjectMetadata{ID: "root"},
TotalSize: int64(2048),
}, int64(8), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
@@ -121,24 +124,31 @@ func TestBackup(t *testing.T) {
t.Helper()
assert.Equal(t, "snap-001", info.ID)
assert.Equal(t, int64(8), info.IncrementalSize)
assert.Positive(t, info.Size)
assert.Equal(t, int64(2048), info.SnapshotSize)
assert.Equal(t, int64(len("test-block-data")), info.SourceSize)
},
},
{
name: "success with CBT",
name: "success with CBT and snapshotSize equal to sourceSize",
setupOpenDev: func(t *testing.T) *os.File {
t.Helper()
return tempFile(t, "test-block-data")
},
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil)
Return(udmrepo.Snapshot{
RootObject: udmrepo.ObjectMetadata{ID: "root"},
TotalSize: int64(len("test-block-data")),
}, int64(8), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
checkInfo: func(t *testing.T, info uploader.SnapshotInfo) {
t.Helper()
assert.Equal(t, "snap-001", info.ID)
assert.Equal(t, int64(8), info.IncrementalSize)
assert.Equal(t, int64(len("test-block-data")), info.SnapshotSize)
assert.Equal(t, int64(len("test-block-data")), info.SourceSize)
},
},
}
@@ -199,12 +209,13 @@ func TestSnapshotSource(t *testing.T) {
baseSource := sourceInfo{realSource: "/test/vol", size: 1024}
testCases := []struct {
name string
setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo)
expectedErrStr string
expectedSnapID string
expectedSize int64
cbtService func(t *testing.T) cbtservice.Service
name string
setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo)
expectedErrStr string
expectedSnapID string
expectedSize int64
expectedSnapshotSize int64
cbtService func(t *testing.T) cbtservice.Service
}{
{
name: "uploader Backup error",
@@ -241,18 +252,19 @@ func TestSnapshotSource(t *testing.T) {
// In full mode, the iterator should cover the whole range if it's a full backup
return iter != nil
}), mock.Anything).
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(512), nil)
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}, TotalSize: 2048}, int64(512), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-success"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
expectedSnapID: "snap-success",
expectedSize: 512,
expectedSnapID: "snap-success",
expectedSize: 512,
expectedSnapshotSize: 2048,
},
{
name: "tags from cbtSource and snapshotTags are merged onto snapshot",
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{}, int64(0), nil)
Return(udmrepo.Snapshot{TotalSize: 4096}, int64(256), nil)
repo.On("SaveSnapshot", mock.Anything, mock.MatchedBy(func(snap udmrepo.Snapshot) bool {
return snap.Tags[uploader.CBTChangeIDTag] == "cid-1" &&
snap.Tags[uploader.CBTVolumeIDTag] == "vid-1" &&
@@ -261,7 +273,9 @@ func TestSnapshotSource(t *testing.T) {
})).Return(udmrepo.ID("snap-tags"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
expectedSnapID: "snap-tags",
expectedSnapID: "snap-tags",
expectedSize: 256,
expectedSnapshotSize: 4096,
},
{
name: "success with cbtService getting allocated blocks",
@@ -277,12 +291,13 @@ func TestSnapshotSource(t *testing.T) {
},
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(1024), nil)
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}, TotalSize: 8192}, int64(1024), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-alloc"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
expectedSnapID: "snap-cbt-alloc",
expectedSize: 1024,
expectedSnapID: "snap-cbt-alloc",
expectedSize: 1024,
expectedSnapshotSize: 8192,
},
{
name: "cbtService error falls back to full",
@@ -296,12 +311,13 @@ func TestSnapshotSource(t *testing.T) {
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
// Should be called with parentObject as empty because of fallback
blkup.On("Backup", mock.Anything, udmrepo.ID(""), mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{}, int64(2048), nil)
Return(udmrepo.Snapshot{TotalSize: 1024}, int64(1024), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-fallback"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
expectedSnapID: "snap-cbt-fallback",
expectedSize: 2048,
expectedSnapID: "snap-cbt-fallback",
expectedSize: 1024,
expectedSnapshotSize: 1024,
},
}
@@ -321,7 +337,7 @@ func TestSnapshotSource(t *testing.T) {
cbtSvc = tc.cbtService(t)
}
snapID, size, err := snapshotSource(
snapID, size, snapshotSize, err := snapshotSource(
ctx, mockRepo, mockBlkup,
baseSource,
true, "",
@@ -337,6 +353,7 @@ func TestSnapshotSource(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, tc.expectedSnapID, snapID)
assert.Equal(t, tc.expectedSize, size)
assert.Equal(t, tc.expectedSnapshotSize, snapshotSize)
}
mockBlkup.AssertExpectations(t)
pkg/uploader/provider/block_test.go
+34 -28
View File
@@ -204,20 +204,21 @@ func TestBlockProviderRunBackup(t *testing.T) {
const requestorType = "test-requestor"
testCases := []struct {
name string
path string
realSource string
tags map[string]string
updater uploader.ProgressUpdater
mockBackupResult uploader.SnapshotInfo
mockBackupErr error
expectedID string
expectedSize int64
expectedIncrSize int64
expectError bool
expectedErrStr string
skipMock bool
checkCaptures func(*testing.T, string, map[string]string)
name string
path string
realSource string
tags map[string]string
updater uploader.ProgressUpdater
mockBackupResult uploader.SnapshotInfo
mockBackupErr error
expectedID string
expectedSize int64
expectedIncrSize int64
expectedSourceSize int64
expectError bool
expectedErrStr string
skipMock bool
checkCaptures func(*testing.T, string, map[string]string)
}{
{
name: "nil updater returns error",
@@ -241,12 +242,14 @@ func TestBlockProviderRunBackup(t *testing.T) {
updater: &blockMockProgressUpdater{},
mockBackupResult: uploader.SnapshotInfo{
ID: "snap-001",
Size: 1024,
SnapshotSize: 2048,
IncrementalSize: 512,
SourceSize: 1024,
},
expectedID: "snap-001",
expectedSize: 1024,
expectedIncrSize: 512,
expectedID: "snap-001",
expectedSize: 2048,
expectedIncrSize: 512,
expectedSourceSize: 1024,
},
{
name: "canceled backup returns ErrorCanceled with partial snapshot info",
@@ -254,15 +257,17 @@ func TestBlockProviderRunBackup(t *testing.T) {
updater: &FakeBackupProgressUpdater{},
mockBackupResult: uploader.SnapshotInfo{
ID: "snap-canceled",
Size: 2048,
SnapshotSize: 2048,
IncrementalSize: 1024,
SourceSize: 1024,
},
mockBackupErr: block.ErrCanceled,
expectedID: "snap-canceled",
expectedSize: 2048,
expectedIncrSize: 1024,
expectError: true,
expectedErrStr: "uploader is canceled",
mockBackupErr: block.ErrCanceled,
expectedID: "snap-canceled",
expectedSize: 2048,
expectedIncrSize: 1024,
expectedSourceSize: 1024,
expectError: true,
expectedErrStr: "uploader is canceled",
},
{
name: "generic backup error is wrapped",
@@ -332,7 +337,7 @@ func TestBlockProviderRunBackup(t *testing.T) {
log: logrus.New(),
}
snapshotID, isEmpty, size, incrSize, err := bp.RunBackup(
snapshotID, isEmpty, size, incrSize, sourceSize, err := bp.RunBackup(
t.Context(),
tc.path,
tc.realSource,
@@ -348,6 +353,7 @@ func TestBlockProviderRunBackup(t *testing.T) {
assert.Equal(t, tc.expectedID, snapshotID)
assert.Equal(t, tc.expectedSize, size)
assert.Equal(t, tc.expectedIncrSize, incrSize)
assert.Equal(t, tc.expectedSourceSize, sourceSize)
if tc.expectError {
require.Error(t, err)
@@ -386,7 +392,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) {
orig := blockBackupFunc
defer func() { blockBackupFunc = orig }()
blockBackupFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ cbtservice.SourceInfo, _ bool, _ string, _ cbtservice.Service, _ map[string]string, _ map[string]string, _ logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) {
return uploader.SnapshotInfo{ID: "snap-cancel", Size: 2048, IncrementalSize: 1024}, false,
return uploader.SnapshotInfo{ID: "snap-cancel", SnapshotSize: 2048, IncrementalSize: 1024, SourceSize: 1024}, false,
errors.Wrapf(
errors.Wrapf(block.ErrCanceled, "error backing up bdev %s", "ns/pvc"),
"Failed to run uploader backup for si %v", "si")
@@ -398,7 +404,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) {
log: logrus.New(),
}
_, _, _, _, err := bp.RunBackup(
_, _, _, _, _, err := bp.RunBackup(
t.Context(), "/dev/sda", "ns/pvc", map[string]string{}, false, "",
CBTParam{}, uploader.PersistentVolumeBlock, map[string]string{},
&FakeBackupProgressUpdater{},
pkg/uploader/provider/kopia_test.go
+1 -1
View File
@@ -106,7 +106,7 @@ func TestRunBackup(t *testing.T) {
tc.volMode = uploader.PersistentVolumeFilesystem
}
kopiaBackupFunc = tc.hookBackupFunc
_, _, _, _, err := kp.RunBackup(t.Context(), "var", "", nil, false, "", CBTParam{}, tc.volMode, map[string]string{}, &updater)
_, _, _, _, _, err := kp.RunBackup(t.Context(), "var", "", nil, false, "", CBTParam{}, tc.volMode, map[string]string{}, &updater)
if tc.notError {
assert.NoError(t, err)
} else {
pkg/uploader/provider/mocks/Provider.go
+15 -9
View File
@@ -91,7 +91,7 @@ func (_c *Provider_Close_Call) RunAndReturn(run func(ctx context.Context) error)
}
// RunBackup provides a mock function for the type Provider
func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error) {
func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error) {
ret := _mock.Called(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater)
if len(ret) == 0 {
@@ -102,8 +102,9 @@ func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource st
var r1 bool
var r2 int64
var r3 int64
var r4 error
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, int64, int64, error)); ok {
var r4 int64
var r5 error
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, int64, int64, int64, error)); ok {
return returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) string); ok {
@@ -126,12 +127,17 @@ func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource st
} else {
r3 = ret.Get(3).(int64)
}
if returnFunc, ok := ret.Get(4).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
if returnFunc, ok := ret.Get(4).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok {
r4 = returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater)
} else {
r4 = ret.Error(4)
r4 = ret.Get(4).(int64)
}
return r0, r1, r2, r3, r4
if returnFunc, ok := ret.Get(5).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
r5 = returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater)
} else {
r5 = ret.Error(5)
}
return r0, r1, r2, r3, r4, r5
}
// Provider_RunBackup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunBackup'
@@ -212,12 +218,12 @@ func (_c *Provider_RunBackup_Call) Run(run func(ctx context.Context, path string
return _c
}
func (_c *Provider_RunBackup_Call) Return(s string, b bool, n int64, n1 int64, err error) *Provider_RunBackup_Call {
_c.Call.Return(s, b, n, n1, err)
func (_c *Provider_RunBackup_Call) Return(_a0 string, _a1 bool, _a2 int64, _a3 int64, _a4 int64, _a5 error) *Provider_RunBackup_Call {
_c.Call.Return(_a0, _a1, _a2, _a3, _a4, _a5)
return _c
}
func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error)) *Provider_RunBackup_Call {
func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error)) *Provider_RunBackup_Call {
_c.Call.Return(run)
return _c
}