mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-13 11:34:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #10506 from Lyndon-Li/save-source-size-to-backup

Browse Source 
Save source size to volume info
This commit is contained in:
lyndon-li
2026-09-10 14:29:48 +08:00
committed by GitHub
parent e8af012ac5 daed42b1d8
commit 87b45ed7fd
22 changed files with 182 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/velero/v1/pod_volume_backup_types.go
+4
View File
@@ -132,6 +132,10 @@ type PodVolumeBackupStatus struct {
// +optional
IncrementalBytes *int64 `json:"incrementalBytes,omitempty"`
// SourceSize holds the total size of the source volume.
// +optional
SourceSize int64 `json:"sourceSize,omitempty"`
// AcceptedTimestamp records the time the pod volume backup is to be prepared.
// The server's time is used for AcceptedTimestamp
// +optional
pkg/apis/velero/v2alpha1/data_upload_types.go
+4
View File
@@ -177,6 +177,10 @@ type DataUploadStatus struct {
// +optional
IncrementalBytes *int64 `json:"incrementalBytes,omitempty"`
// SourceSize holds the total size of the source volume.
// +optional
SourceSize int64 `json:"sourceSize,omitempty"`
// Node is name of the node where the DataUpload is processed.
// +optional
Node string `json:"node,omitempty"`
pkg/backup/backup.go
+1
View File
@@ -1328,6 +1328,7 @@ func updateVolumeInfos(
volumeInfos[index].SnapshotDataMovementInfo.RetainedSnapshot = dataUpload.Spec.CSISnapshot.VolumeSnapshot
volumeInfos[index].SnapshotDataMovementInfo.Size = dataUpload.Status.Progress.TotalBytes
volumeInfos[index].SnapshotDataMovementInfo.IncrementalSize = dataUpload.Status.IncrementalBytes
volumeInfos[index].SnapshotDataMovementInfo.SourceSize = dataUpload.Status.SourceSize
volumeInfos[index].SnapshotDataMovementInfo.Phase = dataUpload.Status.Phase
if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCompleted {
pkg/controller/data_upload_controller.go
+2
View File
@@ -513,6 +513,8 @@ func (r *DataUploadReconciler) OnDataUploadCompleted(ctx context.Context, namesp
du.Status.Phase = velerov2alpha1api.DataUploadPhaseCompleted
du.Status.SnapshotID = result.Backup.SnapshotID
du.Status.IncrementalBytes = result.Backup.IncrementalBytes
du.Status.SourceSize = result.Backup.SourceSize
du.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()}
if result.Backup.EmptySnapshot {
du.Status.Message = "volume was empty so no data was upload"
pkg/controller/pod_volume_backup_controller.go
+1
View File
@@ -550,6 +550,7 @@ func (r *PodVolumeBackupReconciler) OnDataPathCompleted(ctx context.Context, nam
pvb.Status.SnapshotID = result.Backup.SnapshotID
pvb.Status.CompletionTimestamp = &completionTime
pvb.Status.IncrementalBytes = result.Backup.IncrementalBytes
pvb.Status.SourceSize = result.Backup.SourceSize
if result.Backup.EmptySnapshot {
pvb.Status.Message = "volume was empty so no snapshot was taken"
}
pkg/datamover/backup_micro_service_test.go
+1 -1
View File
@@ -152,7 +152,7 @@ func TestOnDataUploadCompleted(t *testing.T) {
{
name: "marshal fail",
marshalErr: errors.New("fake-marshal-error"),
expectedErr: "Failed to marshal backup result { false { } 0 <nil>}: fake-marshal-error",
expectedErr: "Failed to marshal backup result { false { } 0 <nil> 0}: fake-marshal-error",
},
{
name: "succeed",
pkg/datapath/data_path.go
+9 -2
View File
@@ -197,7 +197,7 @@ func (dp *generalDataPath) StartBackup(source AccessPoint, uploaderConfig map[st
dp.wgDataPath.Done()
}()
snapshotID, emptySnapshot, totalBytes, incrementalBytes, err := dp.uploaderProv.RunBackup(
snapshotID, emptySnapshot, totalBytes, incrementalBytes, sourceSize, err := dp.uploaderProv.RunBackup(
dp.ctx,
source.ByPath,
backupParam.RealSource,
@@ -226,7 +226,14 @@ func (dp *generalDataPath) StartBackup(source AccessPoint, uploaderConfig map[st
}
dp.callbacks.OnFailed(context.Background(), dp.namespace, dp.jobName, dataPathErr)
} else {
dp.callbacks.OnCompleted(context.Background(), dp.namespace, dp.jobName, Result{Backup: BackupResult{snapshotID, emptySnapshot, source, totalBytes, ptr.To(incrementalBytes)}})
dp.callbacks.OnCompleted(context.Background(), dp.namespace, dp.jobName, Result{Backup: BackupResult{
SnapshotID: snapshotID,
EmptySnapshot: emptySnapshot,
Source: source,
TotalBytes: totalBytes,
IncrementalBytes: ptr.To(incrementalBytes),
SourceSize: sourceSize,
}})
}
}()
pkg/datapath/data_path_test.go
+28 -4
View File
@@ -71,7 +71,7 @@ func TestAsyncBackup(t *testing.T) {
err: provider.ErrorCanceled,
},
{
name: "async backup complete",
name: "async backup complete with totalBytes larger than sourceSize",
callbacks: Callbacks{
OnFailed: nil,
OnCancelled: nil,
@@ -86,8 +86,32 @@ func TestAsyncBackup(t *testing.T) {
SnapshotID: "fake-snapshot",
EmptySnapshot: false,
Source: AccessPoint{ByPath: "fake-path"},
TotalBytes: 1000,
IncrementalBytes: ptr.To(int64(0)),
TotalBytes: 3000,
IncrementalBytes: ptr.To(int64(200)),
SourceSize: 2000,
},
},
path: "fake-path",
},
{
name: "async backup complete with totalBytes equal to sourceSize",
callbacks: Callbacks{
OnFailed: nil,
OnCancelled: nil,
OnCompleted: func(ctx context.Context, namespace string, job string, result Result) {
asyncResult = result
asyncErr = nil
finish <- struct{}{}
},
},
result: Result{
Backup: BackupResult{
SnapshotID: "fake-snapshot",
EmptySnapshot: false,
Source: AccessPoint{ByPath: "fake-path"},
TotalBytes: 2000,
IncrementalBytes: ptr.To(int64(200)),
SourceSize: 2000,
},
},
path: "fake-path",
@@ -102,7 +126,7 @@ func TestAsyncBackup(t *testing.T) {
if test.result.Backup.IncrementalBytes != nil {
incrementalBytes = *test.result.Backup.IncrementalBytes
}
mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, incrementalBytes, test.err)
mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, incrementalBytes, test.result.Backup.SourceSize, test.err)
mockProvider.On("Close", mock.Anything).Return(nil)
dp.uploaderProv = mockProvider
dp.initialized = true
pkg/datapath/types.go
+1
View File
@@ -39,6 +39,7 @@ type BackupResult struct {
// while a current mover reporting a genuine zero still serializes the key and
// unmarshals to a non-nil zero, distinguishing "measured zero" from "not measured".
IncrementalBytes *int64 `json:"incrementalBytes,omitempty"`
SourceSize int64 `json:"sourceSize,omitempty"`
}
// RestoreResult represents the result of a restore
pkg/podvolume/backup_micro_service_test.go
+1 -1
View File
@@ -156,7 +156,7 @@ func TestOnDataPathCompleted(t *testing.T) {
{
name: "marshal fail",
marshalErr: errors.New("fake-marshal-error"),
expectedErr: "Failed to marshal backup result { false { } 0 <nil>}: fake-marshal-error",
expectedErr: "Failed to marshal backup result { false { } 0 <nil> 0}: fake-marshal-error",
},
{
name: "succeed",
pkg/uploader/block/snapshot.go
+8 -7
View File
@@ -83,11 +83,12 @@ func Backup(ctx context.Context, blkUp Uploader, repoWriter udmrepo.BackupRepo,
return uploader.SnapshotInfo{}, false, errors.Wrapf(err, "error reset pos of block device %s", source)
}
snapID, backupSize, err := snapshotSource(ctx, repoWriter, blkUp, sourceInfo, forceFull, parentSnapshot, cbtSource, cbtService, tags, uploaderCfg, log, "Block Uploader")
snapID, backupSize, snapshotSize, err := snapshotSource(ctx, repoWriter, blkUp, sourceInfo, forceFull, parentSnapshot, cbtSource, cbtService, tags, uploaderCfg, log, "Block Uploader")
snapshotInfo := uploader.SnapshotInfo{
ID: snapID,
Size: sourceInfo.size,
SnapshotSize: snapshotSize,
IncrementalSize: backupSize,
SourceSize: sourceInfo.size,
}
return snapshotInfo, false, err
@@ -106,7 +107,7 @@ func snapshotSource(
uploaderCfg map[string]string,
log logrus.FieldLogger,
description string,
) (string, int64, error) {
) (string, int64, int64, error) {
log.Info("Start to snapshot...")
snapshotStartTime := time.Now()
@@ -128,7 +129,7 @@ func snapshotSource(
snap, backupSize, err := u.Backup(source, parentBackup.parentObject, bitmap.Iterator(), uploaderCfg)
if err != nil {
return "", 0, errors.Wrapf(err, "Failed to run uploader backup for si %v", source)
return "", 0, 0, errors.Wrapf(err, "Failed to run uploader backup for si %v", source)
}
if snap.Tags == nil {
@@ -145,16 +146,16 @@ func snapshotSource(
snapID, err := rep.SaveSnapshot(ctx, snap)
if err != nil {
return "", 0, errors.Wrapf(err, "Failed to save snapshot %v", snap)
return "", 0, 0, errors.Wrapf(err, "Failed to save snapshot %v", snap)
}
if err = rep.Flush(ctx); err != nil {
return "", 0, errors.Wrapf(err, "Failed to flush repository")
return "", 0, 0, errors.Wrapf(err, "Failed to flush repository")
}
log.Infof("Created snapshot with root %v and ID %v in %v", snap.RootObject, snapID, time.Since(snapshotStartTime).Truncate(time.Second))
return string(snapID), backupSize, nil
return string(snapID), backupSize, snap.TotalSize, nil
}
func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull bool, parentSnapshot string, volumeID string,
pkg/uploader/block/snapshot_test.go
+40 -23
View File
@@ -106,14 +106,17 @@ func TestBackup(t *testing.T) {
expectedErrStr: "Failed to run uploader backup",
},
{
name: "success returns correct SnapshotInfo",
name: "success returns correct SnapshotInfo with snapshotSize larger than sourceSize",
setupOpenDev: func(t *testing.T) *os.File {
t.Helper()
return tempFile(t, "test-block-data")
},
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil)
Return(udmrepo.Snapshot{
RootObject: udmrepo.ObjectMetadata{ID: "root"},
TotalSize: int64(2048),
}, int64(8), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
@@ -121,24 +124,31 @@ func TestBackup(t *testing.T) {
t.Helper()
assert.Equal(t, "snap-001", info.ID)
assert.Equal(t, int64(8), info.IncrementalSize)
assert.Positive(t, info.Size)
assert.Equal(t, int64(2048), info.SnapshotSize)
assert.Equal(t, int64(len("test-block-data")), info.SourceSize)
},
},
{
name: "success with CBT",
name: "success with CBT and snapshotSize equal to sourceSize",
setupOpenDev: func(t *testing.T) *os.File {
t.Helper()
return tempFile(t, "test-block-data")
},
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil)
Return(udmrepo.Snapshot{
RootObject: udmrepo.ObjectMetadata{ID: "root"},
TotalSize: int64(len("test-block-data")),
}, int64(8), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
checkInfo: func(t *testing.T, info uploader.SnapshotInfo) {
t.Helper()
assert.Equal(t, "snap-001", info.ID)
assert.Equal(t, int64(8), info.IncrementalSize)
assert.Equal(t, int64(len("test-block-data")), info.SnapshotSize)
assert.Equal(t, int64(len("test-block-data")), info.SourceSize)
},
},
}
@@ -199,12 +209,13 @@ func TestSnapshotSource(t *testing.T) {
baseSource := sourceInfo{realSource: "/test/vol", size: 1024}
testCases := []struct {
name string
setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo)
expectedErrStr string
expectedSnapID string
expectedSize int64
cbtService func(t *testing.T) cbtservice.Service
name string
setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo)
expectedErrStr string
expectedSnapID string
expectedSize int64
expectedSnapshotSize int64
cbtService func(t *testing.T) cbtservice.Service
}{
{
name: "uploader Backup error",
@@ -241,18 +252,19 @@ func TestSnapshotSource(t *testing.T) {
// In full mode, the iterator should cover the whole range if it's a full backup
return iter != nil
}), mock.Anything).
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(512), nil)
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}, TotalSize: 2048}, int64(512), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-success"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
expectedSnapID: "snap-success",
expectedSize: 512,
expectedSnapID: "snap-success",
expectedSize: 512,
expectedSnapshotSize: 2048,
},
{
name: "tags from cbtSource and snapshotTags are merged onto snapshot",
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{}, int64(0), nil)
Return(udmrepo.Snapshot{TotalSize: 4096}, int64(256), nil)
repo.On("SaveSnapshot", mock.Anything, mock.MatchedBy(func(snap udmrepo.Snapshot) bool {
return snap.Tags[uploader.CBTChangeIDTag] == "cid-1" &&
snap.Tags[uploader.CBTVolumeIDTag] == "vid-1" &&
@@ -261,7 +273,9 @@ func TestSnapshotSource(t *testing.T) {
})).Return(udmrepo.ID("snap-tags"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
expectedSnapID: "snap-tags",
expectedSnapID: "snap-tags",
expectedSize: 256,
expectedSnapshotSize: 4096,
},
{
name: "success with cbtService getting allocated blocks",
@@ -277,12 +291,13 @@ func TestSnapshotSource(t *testing.T) {
},
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(1024), nil)
Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}, TotalSize: 8192}, int64(1024), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-alloc"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
expectedSnapID: "snap-cbt-alloc",
expectedSize: 1024,
expectedSnapID: "snap-cbt-alloc",
expectedSize: 1024,
expectedSnapshotSize: 8192,
},
{
name: "cbtService error falls back to full",
@@ -296,12 +311,13 @@ func TestSnapshotSource(t *testing.T) {
setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
// Should be called with parentObject as empty because of fallback
blkup.On("Backup", mock.Anything, udmrepo.ID(""), mock.Anything, mock.Anything).
Return(udmrepo.Snapshot{}, int64(2048), nil)
Return(udmrepo.Snapshot{TotalSize: 1024}, int64(1024), nil)
repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-fallback"), nil)
repo.On("Flush", mock.Anything).Return(nil)
},
expectedSnapID: "snap-cbt-fallback",
expectedSize: 2048,
expectedSnapID: "snap-cbt-fallback",
expectedSize: 1024,
expectedSnapshotSize: 1024,
},
}
@@ -321,7 +337,7 @@ func TestSnapshotSource(t *testing.T) {
cbtSvc = tc.cbtService(t)
}
snapID, size, err := snapshotSource(
snapID, size, snapshotSize, err := snapshotSource(
ctx, mockRepo, mockBlkup,
baseSource,
true, "",
@@ -337,6 +353,7 @@ func TestSnapshotSource(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, tc.expectedSnapID, snapID)
assert.Equal(t, tc.expectedSize, size)
assert.Equal(t, tc.expectedSnapshotSize, snapshotSize)
}
mockBlkup.AssertExpectations(t)
pkg/uploader/kopia/snapshot.go
+3 -2
View File
@@ -192,8 +192,9 @@ func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.Re
snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, uploaderCfg, updater, log, "Kopia Uploader")
snapshotInfo := &uploader.SnapshotInfo{
ID: snapID,
Size: snapshotSize,
ID: snapID,
SnapshotSize: snapshotSize,
SourceSize: snapshotSize,
}
return snapshotInfo, false, err
pkg/uploader/provider/block.go
+9 -9
View File
@@ -103,13 +103,13 @@ func (bp *blockProvider) RunBackup(
cbtParam CBTParam,
volMode uploader.PersistentVolumeMode,
uploaderCfg map[string]string,
updater uploader.ProgressUpdater) (string, bool, int64, int64, error) {
updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error) {
if updater == nil {
return "", false, 0, 0, errors.New("backup progress updater is invalid")
return "", false, 0, 0, 0, errors.New("backup progress updater is invalid")
}
if path == "" {
return "", false, 0, 0, errors.New("path is empty")
return "", false, 0, 0, 0, errors.New("path is empty")
}
log := bp.log.WithFields(logrus.Fields{
@@ -140,23 +140,23 @@ func (bp *blockProvider) RunBackup(
// equality check never matches and cancellation gets reported as a failure.
if errors.Is(err, block.ErrCanceled) {
log.Warn("Block backup is canceled")
return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, ErrorCanceled
return snapshotInfo.ID, false, snapshotInfo.SnapshotSize, snapshotInfo.IncrementalSize, snapshotInfo.SourceSize, ErrorCanceled
}
if err != nil {
return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, errors.Wrapf(err, "Failed to run block backup")
return snapshotInfo.ID, false, snapshotInfo.SnapshotSize, snapshotInfo.IncrementalSize, snapshotInfo.SourceSize, errors.Wrapf(err, "Failed to run block backup")
}
updater.UpdateProgress(
&uploader.Progress{
TotalBytes: snapshotInfo.Size,
BytesDone: snapshotInfo.Size,
TotalBytes: snapshotInfo.SnapshotSize,
BytesDone: snapshotInfo.SnapshotSize,
},
)
log.Infof("Block backup finished, snapshot ID %s, backup size %v, incremental size %v", snapshotInfo.ID, snapshotInfo.Size, snapshotInfo.IncrementalSize)
log.Infof("Block backup finished, snapshot ID %s, backup size %v, incremental size %v, source size %v", snapshotInfo.ID, snapshotInfo.SnapshotSize, snapshotInfo.IncrementalSize, snapshotInfo.SourceSize)
return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, nil
return snapshotInfo.ID, false, snapshotInfo.SnapshotSize, snapshotInfo.IncrementalSize, snapshotInfo.SourceSize, nil
}
func (bp *blockProvider) RunRestore(
pkg/uploader/provider/block_test.go
+34 -28
View File
@@ -204,20 +204,21 @@ func TestBlockProviderRunBackup(t *testing.T) {
const requestorType = "test-requestor"
testCases := []struct {
name string
path string
realSource string
tags map[string]string
updater uploader.ProgressUpdater
mockBackupResult uploader.SnapshotInfo
mockBackupErr error
expectedID string
expectedSize int64
expectedIncrSize int64
expectError bool
expectedErrStr string
skipMock bool
checkCaptures func(*testing.T, string, map[string]string)
name string
path string
realSource string
tags map[string]string
updater uploader.ProgressUpdater
mockBackupResult uploader.SnapshotInfo
mockBackupErr error
expectedID string
expectedSize int64
expectedIncrSize int64
expectedSourceSize int64
expectError bool
expectedErrStr string
skipMock bool
checkCaptures func(*testing.T, string, map[string]string)
}{
{
name: "nil updater returns error",
@@ -241,12 +242,14 @@ func TestBlockProviderRunBackup(t *testing.T) {
updater: &blockMockProgressUpdater{},
mockBackupResult: uploader.SnapshotInfo{
ID: "snap-001",
Size: 1024,
SnapshotSize: 2048,
IncrementalSize: 512,
SourceSize: 1024,
},
expectedID: "snap-001",
expectedSize: 1024,
expectedIncrSize: 512,
expectedID: "snap-001",
expectedSize: 2048,
expectedIncrSize: 512,
expectedSourceSize: 1024,
},
{
name: "canceled backup returns ErrorCanceled with partial snapshot info",
@@ -254,15 +257,17 @@ func TestBlockProviderRunBackup(t *testing.T) {
updater: &FakeBackupProgressUpdater{},
mockBackupResult: uploader.SnapshotInfo{
ID: "snap-canceled",
Size: 2048,
SnapshotSize: 2048,
IncrementalSize: 1024,
SourceSize: 1024,
},
mockBackupErr: block.ErrCanceled,
expectedID: "snap-canceled",
expectedSize: 2048,
expectedIncrSize: 1024,
expectError: true,
expectedErrStr: "uploader is canceled",
mockBackupErr: block.ErrCanceled,
expectedID: "snap-canceled",
expectedSize: 2048,
expectedIncrSize: 1024,
expectedSourceSize: 1024,
expectError: true,
expectedErrStr: "uploader is canceled",
},
{
name: "generic backup error is wrapped",
@@ -332,7 +337,7 @@ func TestBlockProviderRunBackup(t *testing.T) {
log: logrus.New(),
}
snapshotID, isEmpty, size, incrSize, err := bp.RunBackup(
snapshotID, isEmpty, size, incrSize, sourceSize, err := bp.RunBackup(
t.Context(),
tc.path,
tc.realSource,
@@ -348,6 +353,7 @@ func TestBlockProviderRunBackup(t *testing.T) {
assert.Equal(t, tc.expectedID, snapshotID)
assert.Equal(t, tc.expectedSize, size)
assert.Equal(t, tc.expectedIncrSize, incrSize)
assert.Equal(t, tc.expectedSourceSize, sourceSize)
if tc.expectError {
require.Error(t, err)
@@ -386,7 +392,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) {
orig := blockBackupFunc
defer func() { blockBackupFunc = orig }()
blockBackupFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ cbtservice.SourceInfo, _ bool, _ string, _ cbtservice.Service, _ map[string]string, _ map[string]string, _ logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) {
return uploader.SnapshotInfo{ID: "snap-cancel", Size: 2048, IncrementalSize: 1024}, false,
return uploader.SnapshotInfo{ID: "snap-cancel", SnapshotSize: 2048, IncrementalSize: 1024, SourceSize: 1024}, false,
errors.Wrapf(
errors.Wrapf(block.ErrCanceled, "error backing up bdev %s", "ns/pvc"),
"Failed to run uploader backup for si %v", "si")
@@ -398,7 +404,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) {
log: logrus.New(),
}
_, _, _, _, err := bp.RunBackup(
_, _, _, _, _, err := bp.RunBackup(
t.Context(), "/dev/sda", "ns/pvc", map[string]string{}, false, "",
CBTParam{}, uploader.PersistentVolumeBlock, map[string]string{},
&FakeBackupProgressUpdater{},
pkg/uploader/provider/kopia.go
+9 -9
View File
@@ -121,13 +121,13 @@ func (kp *kopiaProvider) RunBackup(
volMode uploader.PersistentVolumeMode,
uploaderCfg map[string]string,
updater uploader.ProgressUpdater,
) (string, bool, int64, int64, error) {
) (string, bool, int64, int64, int64, error) {
if updater == nil {
return "", false, 0, 0, errors.New("Need to initial backup progress updater first")
return "", false, 0, 0, 0, errors.New("Need to initial backup progress updater first")
}
if path == "" {
return "", false, 0, 0, errors.New("path is empty")
return "", false, 0, 0, 0, errors.New("path is empty")
}
log := kp.log.WithFields(logrus.Fields{
@@ -177,21 +177,21 @@ func (kp *kopiaProvider) RunBackup(
if kpUploader.IsCanceled() {
log.Warn("Kopia backup is canceled")
return snapshotID, false, 0, 0, ErrorCanceled
return snapshotID, false, 0, 0, 0, ErrorCanceled
}
return snapshotID, false, 0, 0, errors.Wrapf(err, "Failed to run kopia backup")
return snapshotID, false, 0, 0, 0, errors.Wrapf(err, "Failed to run kopia backup")
}
// which ensure that the statistic data of TotalBytes equal to BytesDone when finished
updater.UpdateProgress(
&uploader.Progress{
TotalBytes: snapshotInfo.Size,
BytesDone: snapshotInfo.Size,
TotalBytes: snapshotInfo.SnapshotSize,
BytesDone: snapshotInfo.SnapshotSize,
},
)
log.Debugf("Kopia backup finished, snapshot ID %s, backup size %d", snapshotInfo.ID, snapshotInfo.Size)
return snapshotInfo.ID, false, snapshotInfo.Size, progress.GetIncrementalSize(), nil
log.Debugf("Kopia backup finished, snapshot ID %s, backup size %d", snapshotInfo.ID, snapshotInfo.SnapshotSize)
return snapshotInfo.ID, false, snapshotInfo.SnapshotSize, progress.GetIncrementalSize(), snapshotInfo.SourceSize, nil
}
func (kp *kopiaProvider) GetPassword(param any) (string, error) {
pkg/uploader/provider/kopia_test.go
+1 -1
View File
@@ -106,7 +106,7 @@ func TestRunBackup(t *testing.T) {
tc.volMode = uploader.PersistentVolumeFilesystem
}
kopiaBackupFunc = tc.hookBackupFunc
_, _, _, _, err := kp.RunBackup(t.Context(), "var", "", nil, false, "", CBTParam{}, tc.volMode, map[string]string{}, &updater)
_, _, _, _, _, err := kp.RunBackup(t.Context(), "var", "", nil, false, "", CBTParam{}, tc.volMode, map[string]string{}, &updater)
if tc.notError {
assert.NoError(t, err)
} else {
pkg/uploader/provider/mocks/Provider.go
+15 -9
View File
@@ -91,7 +91,7 @@ func (_c *Provider_Close_Call) RunAndReturn(run func(ctx context.Context) error)
}
// RunBackup provides a mock function for the type Provider
func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error) {
func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error) {
ret := _mock.Called(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater)
if len(ret) == 0 {
@@ -102,8 +102,9 @@ func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource st
var r1 bool
var r2 int64
var r3 int64
var r4 error
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, int64, int64, error)); ok {
var r4 int64
var r5 error
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, int64, int64, int64, error)); ok {
return returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater)
}
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) string); ok {
@@ -126,12 +127,17 @@ func (_mock *Provider) RunBackup(ctx context.Context, path string, realSource st
} else {
r3 = ret.Get(3).(int64)
}
if returnFunc, ok := ret.Get(4).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
if returnFunc, ok := ret.Get(4).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok {
r4 = returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater)
} else {
r4 = ret.Error(4)
r4 = ret.Get(4).(int64)
}
return r0, r1, r2, r3, r4
if returnFunc, ok := ret.Get(5).(func(context.Context, string, string, map[string]string, bool, string, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
r5 = returnFunc(ctx, path, realSource, tags, forceFull, parentSnapshot, cbtParam, volMode, uploaderCfg, updater)
} else {
r5 = ret.Error(5)
}
return r0, r1, r2, r3, r4, r5
}
// Provider_RunBackup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunBackup'
@@ -212,12 +218,12 @@ func (_c *Provider_RunBackup_Call) Run(run func(ctx context.Context, path string
return _c
}
func (_c *Provider_RunBackup_Call) Return(s string, b bool, n int64, n1 int64, err error) *Provider_RunBackup_Call {
_c.Call.Return(s, b, n, n1, err)
func (_c *Provider_RunBackup_Call) Return(_a0 string, _a1 bool, _a2 int64, _a3 int64, _a4 int64, _a5 error) *Provider_RunBackup_Call {
_c.Call.Return(_a0, _a1, _a2, _a3, _a4, _a5)
return _c
}
func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error)) *Provider_RunBackup_Call {
func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error)) *Provider_RunBackup_Call {
_c.Call.Return(run)
return _c
}
pkg/uploader/provider/provider.go
+1 -1
View File
@@ -57,7 +57,7 @@ type Provider interface {
cbtParam CBTParam,
volMode uploader.PersistentVolumeMode,
uploaderCfg map[string]string,
updater uploader.ProgressUpdater) (string, bool, int64, int64, error)
updater uploader.ProgressUpdater) (string, bool, int64, int64, int64, error)
// RunRestore which will do restore for one specific volume with given snapshot id and return error
// updater is used for updating backup progress which implement by third-party
RunRestore(
pkg/uploader/types.go
+2 -1
View File
@@ -52,8 +52,9 @@ func ValidateUploaderType(t string) (string, error) {
type SnapshotInfo struct {
ID string
Size int64
SnapshotSize int64
IncrementalSize int64
SourceSize int64
}
// Progress which defined two variables to record progress