Cherry pick the in-place restore implementation PRs from feature branch to main (#10415)

* Update CRDs and CLI to support in-place restore (#10038) Update CRDs(Restore, DataDownload, PodVolumeRestore) and restore create CLI to support in-place restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Kopia(filesystem) uploader to support incremental and deleteExtraFile during restore (#10066) Update Kopia(filesystem) uploader to support incremental and deleteExtraFile during restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Restore Exposer and PVC CSI to support in-place restore (#10104) 1. Update Restore Exposer to support exposing with existing PV for in-place restore 2. Update PVC CSI RIA to continue the restore process for in-place restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Block uploader to support increase restore (#10244) Update Block uploader to support increase restore Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Update Exposer to recreate the target PV if the volume mode is different with the restore PVC (#10257) Update Exposer to recreate the target PV if the volume mode is different with t he restore PVC Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> * Preserve PVC selected-node annotation via carrier annotation for in-place restore For in-place volume data restore, the existing PVC is deleted and recreated. For StorageClasses with the WaitForFirstConsumer volume binding mode, losing the volume.kubernetes.io/selected-node annotation could let the scheduler place the recreated workload Pod in a different zone than the original PV, leaving it stuck in ContainerCreating. Instead of relying on RestoreItemAction execution order (the generic PVC RIA unconditionally strips the selected-node annotation), the PVC CSI RIA now captures the annotation from the existing PVC right before deleting it and carries it on the target PVC via the Velero-internal restore.velero.io/inplace-restore-selected-node annotation. The restore engine translates the carrier back to the Kubernetes annotation after all RestoreItemActions have run and always strips the carrier so it never lands on the cluster. This makes the behavior independent of RIA ordering: the Kubernetes annotation is stripped by default on every path (including when the target PVC does not exist and Velero falls back to provisioning a new PVC), and preservation only happens when the CSI RIA explicitly captured a value from the existing PVC. Signed-off-by: chlins <chlins.zhang@gmail.com> * Update the control path to make the in-place incremental restore with block data mover work E2E (#10410) Update the control path to make the in-place incremental restore with block data mover work E2E Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> --------- Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com> Signed-off-by: chlins <chlins.zhang@gmail.com> Co-authored-by: chlins <chlins.zhang@gmail.com>
2026-09-19 14:34:17 +00:00 · 2026-08-26 10:21:32 -04:00
parent ac5744c7b4
commit b7d83a6f2b
54 changed files with 2164 additions and 335 deletions
@@ -205,18 +205,44 @@ func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull
 }

 // Restore restore specific sourcePath with given snapshotID and update progress
-func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapshotID, dest string, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) {
+func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapshotID, dest string, incremental bool, cbtSource cbtservice.SourceInfo, cbtService cbtservice.Service, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) {
 	log.Info("Start to restore...")

 	snapshot, err := rep.GetSnapshot(ctx, udmrepo.ID(snapshotID))
 	if err != nil {
 		return 0, errors.Wrapf(err, "Unable to load snapshot %v", snapshotID)
 	}
+	log.Infof("Restore from snapshot %s, incremental %v, cbt source %v, description %s, created time %v, tags %v", snapshotID, incremental, cbtSource, snapshot.Description, snapshot.EndTime, snapshot.Tags)

-	log.Infof("Restore from snapshot %s, description %s, created time %v, tags %v", snapshotID, snapshot.Description, snapshot.EndTime, snapshot.Tags)
+	var volumeSnapshot, changeID, volumeID string
+	if incremental {
+		if snapshot.Tags == nil {
+			log.Warnf("No tag from snapshot %s, fallback to full restore", snapshotID)
+			incremental = false
+		} else if snapshot.Tags[uploader.CBTChangeIDTag] == "" {
+			log.Warnf("No ChangeID tag from snapshot %s, fallback to full restore", snapshotID)
+			incremental = false
+		} else if snapshot.Tags[uploader.CBTVolumeIDTag] == "" {
+			log.Warnf("No VolumeID tag from snapshot %s, fallback to full restore", snapshotID)
+			incremental = false
+		} else if snapshot.Tags[uploader.CBTVolumeIDTag] != cbtSource.VolumeID {
+			log.Warnf("VolumeID %s from snapshot %s is not expected as %s, fallback to full restore", snapshot.Tags[uploader.CBTVolumeIDTag], snapshotID, cbtSource.VolumeID)
+			incremental = false
+		} else {
+			volumeSnapshot = cbtSource.Snapshot
+			changeID = snapshot.Tags[uploader.CBTChangeIDTag]
+			volumeID = snapshot.Tags[uploader.CBTVolumeIDTag]
+		}
+	}

-	bitmap := cbt.NewBitmap(blockSize, uint64(snapshot.TotalSize), "", "", "")
-	bitmap.SetFull()
+	bitmap := cbt.NewBitmap(blockSize, uint64(snapshot.TotalSize), volumeSnapshot, changeID, volumeID)
+	if incremental {
+		if err = cbt.SetBitmapOrFull(ctx, cbtService, bitmap); err != nil {
+			log.WithError(err).Warnf("Failed to create CBT with source %v, fallback to full restore", cbtSource)
+		}
+	} else {
+		bitmap.SetFull()
+	}

 	destPath, err := filepath.Abs(dest)
 	if err != nil {
@@ -33,6 +33,7 @@ import (
 	"github.com/stretchr/testify/require"

 	"github.com/vmware-tanzu/velero/pkg/cbtservice"
+	cbtservicemocks "github.com/vmware-tanzu/velero/pkg/cbtservice/mocks"
 	"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
 	udmrepomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks"
 	"github.com/vmware-tanzu/velero/pkg/uploader"
@@ -123,6 +124,23 @@ func TestBackup(t *testing.T) {
 				assert.Positive(t, info.Size)
 			},
 		},
+		{
+			name: "success with CBT",
+			setupOpenDev: func(t *testing.T) *os.File {
+				t.Helper()
+				return tempFile(t, "test-block-data")
+			},
+			setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
+				blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+					Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil)
+				repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil)
+				repo.On("Flush", mock.Anything).Return(nil)
+			},
+			checkInfo: func(t *testing.T, info uploader.SnapshotInfo) {
+				t.Helper()
+				assert.Equal(t, "snap-001", info.ID)
+			},
+		},
 	}

 	for _, tc := range testCases {
@@ -186,6 +204,7 @@ func TestSnapshotSource(t *testing.T) {
 		expectedErrStr string
 		expectedSnapID string
 		expectedSize   int64
+		cbtService     func(t *testing.T) cbtservice.Service
 	}{
 		{
 			name: "uploader Backup error",
@@ -218,7 +237,10 @@ func TestSnapshotSource(t *testing.T) {
 		{
 			name: "success with nil cbtService falls back to full bitmap",
 			setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
-				blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+				blkup.On("Backup", mock.Anything, mock.Anything, mock.MatchedBy(func(iter cbttypes.Iterator) bool {
+					// In full mode, the iterator should cover the whole range if it's a full backup
+					return iter != nil
+				}), mock.Anything).
 					Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(512), nil)
 				repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-success"), nil)
 				repo.On("Flush", mock.Anything).Return(nil)
@@ -241,6 +263,46 @@ func TestSnapshotSource(t *testing.T) {
 			},
 			expectedSnapID: "snap-tags",
 		},
+		{
+			name: "success with cbtService getting allocated blocks",
+			cbtService: func(t *testing.T) cbtservice.Service {
+				t.Helper()
+				m := cbtservicemocks.NewService(t)
+				m.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).
+					Run(func(args mock.Arguments) {
+						record := args.Get(2).(func([]cbtservice.Range) error)
+						record([]cbtservice.Range{{Offset: 0, Length: 1024}})
+					}).Return(nil)
+				return m
+			},
+			setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
+				blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+					Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(1024), nil)
+				repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-alloc"), nil)
+				repo.On("Flush", mock.Anything).Return(nil)
+			},
+			expectedSnapID: "snap-cbt-alloc",
+			expectedSize:   1024,
+		},
+		{
+			name: "cbtService error falls back to full",
+			cbtService: func(t *testing.T) cbtservice.Service {
+				t.Helper()
+				m := cbtservicemocks.NewService(t)
+				m.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).
+					Return(errors.New("CBT error"))
+				return m
+			},
+			setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
+				// Should be called with parentObject as empty because of fallback
+				blkup.On("Backup", mock.Anything, udmrepo.ID(""), mock.Anything, mock.Anything).
+					Return(udmrepo.Snapshot{}, int64(2048), nil)
+				repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-fallback"), nil)
+				repo.On("Flush", mock.Anything).Return(nil)
+			},
+			expectedSnapID: "snap-cbt-fallback",
+			expectedSize:   2048,
+		},
 	}

 	for _, tc := range testCases {
@@ -251,14 +313,19 @@ func TestSnapshotSource(t *testing.T) {

 			tc.setupMocks(mockBlkup, mockRepo)

-			cbtSrc := cbtservice.SourceInfo{ChangeID: "cid-1", VolumeID: "vid-1"}
+			cbtSrc := cbtservice.SourceInfo{Snapshot: "snap-1", ChangeID: "cid-1", VolumeID: "vid-1"}
 			snapshotTags := map[string]string{"custom": "val"}

+			var cbtSvc cbtservice.Service
+			if tc.cbtService != nil {
+				cbtSvc = tc.cbtService(t)
+			}
+
 			snapID, size, err := snapshotSource(
 				ctx, mockRepo, mockBlkup,
 				baseSource,
 				true, "",
-				cbtSrc, nil,
+				cbtSrc, cbtSvc,
 				snapshotTags, map[string]string{},
 				testLog(), "Block Uploader",
 			)
@@ -601,6 +668,9 @@ func TestRestore(t *testing.T) {

 	testCases := []struct {
 		name           string
+		incremental    bool
+		cbtSource      cbtservice.SourceInfo
+		cbtService     func(t *testing.T) cbtservice.Service
 		setupMocks     func(blkup *mockUploader, repo *udmrepomocks.BackupRepo)
 		setupOpenDev   func(t *testing.T) *os.File
 		expectedErrStr string
@@ -637,7 +707,7 @@ func TestRestore(t *testing.T) {
 			expectedErrStr: "error restoring to block dev",
 		},
 		{
-			name: "success returns size",
+			name: "success returns size (full restore)",
 			setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
 				repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).
 					Return(storedSnap, nil)
@@ -650,6 +720,102 @@ func TestRestore(t *testing.T) {
 			},
 			expectedSize: 4096,
 		},
+		{
+			name:        "incremental restore success",
+			incremental: true,
+			cbtSource:   cbtservice.SourceInfo{Snapshot: "snap-cbt", VolumeID: "vol-1"},
+			cbtService: func(t *testing.T) cbtservice.Service {
+				t.Helper()
+				m := cbtservicemocks.NewService(t)
+				m.On("GetChangedBlocks", mock.Anything, "snap-cbt", "cid-1", mock.Anything).
+					Run(func(args mock.Arguments) {
+						record := args.Get(3).(func([]cbtservice.Range) error)
+						record([]cbtservice.Range{{Offset: 0, Length: 512}})
+					}).Return(nil)
+				return m
+			},
+			setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
+				snapWithTags := udmrepo.Snapshot{
+					Tags: map[string]string{
+						uploader.CBTChangeIDTag: "cid-1",
+						uploader.CBTVolumeIDTag: "vol-1",
+					},
+					TotalSize: 1024,
+				}
+				repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(snapWithTags, nil)
+				blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+					Return(int64(512), int64(512), nil)
+			},
+			setupOpenDev: func(t *testing.T) *os.File {
+				t.Helper()
+				return tempFile(t, "")
+			},
+			expectedSize: 512,
+		},
+		{
+			name:        "incremental restore fallback - missing tags",
+			incremental: true,
+			setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
+				repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(storedSnap, nil)
+				blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+					Return(int64(4096), int64(4096), nil)
+			},
+			setupOpenDev: func(t *testing.T) *os.File {
+				t.Helper()
+				return tempFile(t, "")
+			},
+			expectedSize: 4096,
+		},
+		{
+			name:        "incremental restore fallback - VolumeID mismatch",
+			incremental: true,
+			cbtSource:   cbtservice.SourceInfo{VolumeID: "vol-actual"},
+			setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
+				snapWithTags := udmrepo.Snapshot{
+					Tags: map[string]string{
+						uploader.CBTChangeIDTag: "cid-1",
+						uploader.CBTVolumeIDTag: "vol-expected",
+					},
+				}
+				repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(snapWithTags, nil)
+				blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+					Return(int64(4096), int64(4096), nil)
+			},
+			setupOpenDev: func(t *testing.T) *os.File {
+				t.Helper()
+				return tempFile(t, "")
+			},
+			expectedSize: 4096,
+		},
+		{
+			name:        "incremental restore fallback - CBT service error",
+			incremental: true,
+			cbtSource:   cbtservice.SourceInfo{Snapshot: "snap-cbt", VolumeID: "vol-1"},
+			cbtService: func(t *testing.T) cbtservice.Service {
+				t.Helper()
+				m := cbtservicemocks.NewService(t)
+				m.On("GetChangedBlocks", mock.Anything, "snap-cbt", "cid-1", mock.Anything).
+					Return(errors.New("CBT error"))
+				return m
+			},
+			setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) {
+				snapWithTags := udmrepo.Snapshot{
+					Tags: map[string]string{
+						uploader.CBTChangeIDTag: "cid-1",
+						uploader.CBTVolumeIDTag: "vol-1",
+					},
+					TotalSize: 1024,
+				}
+				repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(snapWithTags, nil)
+				blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+					Return(int64(1024), int64(1024), nil)
+			},
+			setupOpenDev: func(t *testing.T) *os.File {
+				t.Helper()
+				return tempFile(t, "")
+			},
+			expectedSize: 1024,
+		},
 	}

 	for _, tc := range testCases {
@@ -671,7 +837,12 @@ func TestRestore(t *testing.T) {
 				}
 			}

-			size, err := Restore(ctx, mockBlkup, mockRepo, "snap-001", "/dev/sdb", map[string]string{}, testLog())
+			var cbtSvc cbtservice.Service
+			if tc.cbtService != nil {
+				cbtSvc = tc.cbtService(t)
+			}
+
+			size, err := Restore(ctx, mockBlkup, mockRepo, "snap-001", "/dev/sdb", tc.incremental, tc.cbtSource, cbtSvc, map[string]string{}, testLog())

 			if tc.expectedErrStr != "" {
 				require.Error(t, err)
@@ -389,7 +389,7 @@ func (o *fileSystemRestoreOutput) Terminate() error {
 }

 // Restore restore specific sourcePath with given snapshotID and update progress
-func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string,
+func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string,
 	log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
 	log.Info("Start to restore...")

@@ -421,7 +421,7 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
 	}

 	restoreConcurrency := runtime.NumCPU()
-
+	deleteExtra := false
 	if len(uploaderCfg) > 0 {
 		writeSparseFiles, err := uploaderutil.GetWriteSparseFiles(uploaderCfg)
 		if err != nil {
@@ -438,9 +438,14 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
 		if concurrency > 0 {
 			restoreConcurrency = concurrency
 		}
+
+		deleteExtra, err = uploaderutil.GetDeleteExtraFiles(uploaderCfg)
+		if err != nil {
+			return 0, 0, errors.Wrap(err, "failed to get delete extra files config")
+		}
 	}

-	log.Debugf("Restore filesystem output %v, concurrency %d", fsOutput, restoreConcurrency)
+	log.Debugf("Restore filesystem output %v, concurrency %d, incremental %v, delete extra %v", fsOutput, restoreConcurrency, incremental, deleteExtra)

 	err = fsOutput.Init(ctx)
 	if err != nil {
@@ -448,14 +453,22 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
 	}

 	var output RestoreOutput
+	// kopiaOutput is the output passed to Kopia's restore.Entry function.
+	// We must pass the unwrapped fsOutput (*restore.FilesystemOutput) directly for file system restores.
+	// This is because Kopia internally uses a strict type assertion (c.output.(*FilesystemOutput))
+	// to determine if it should execute the deleteExtra logic. If we pass the wrapped
+	// fileSystemRestoreOutput, the type assertion fails and extra files are not deleted.
+	var kopiaOutput restore.Output
 	if volMode == uploader.PersistentVolumeBlock {
 		output = &BlockOutput{
 			FilesystemOutput: fsOutput,
 		}
+		kopiaOutput = output
 	} else {
 		output = &fileSystemRestoreOutput{
 			FilesystemOutput: fsOutput,
 		}
+		kopiaOutput = fsOutput
 	}

 	defer func() {
@@ -464,8 +477,10 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
 		}
 	}()

-	stat, err := restoreEntryFunc(kopiaCtx, rep, output, rootEntry, restore.Options{
+	stat, err := restoreEntryFunc(kopiaCtx, rep, kopiaOutput, rootEntry, restore.Options{
 		Parallel:               restoreConcurrency,
+		Incremental:            incremental,
+		DeleteExtra:            deleteExtra,
 		RestoreDirEntryAtDepth: math.MaxInt32,
 		Cancel:                 cancleCh,
 		ProgressCallback: func(ctx context.Context, stats restore.Stats) {
@@ -681,6 +681,7 @@ func TestRestore(t *testing.T) {
 		expectedCount       int32
 		expectedError       error
 		volMode             uploader.PersistentVolumeMode
+		incremental         bool
 	}

 	// Define test cases
@@ -818,7 +819,7 @@ func TestRestore(t *testing.T) {
 			repoWriterMock.On("OpenObject", mock.Anything, mock.Anything).Return(em, nil)

 			progress := new(Progress)
-			bytesRestored, fileCount, err := Restore(t.Context(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, map[string]string{}, logrus.New(), nil)
+			bytesRestored, fileCount, err := Restore(t.Context(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.incremental, tc.volMode, map[string]string{}, logrus.New(), nil)

 			// Check if the returned error matches the expected error
 			if tc.expectedError != nil {
@@ -163,6 +163,8 @@ func (bp *blockProvider) RunRestore(
 	ctx context.Context,
 	snapshotID string,
 	volumePath string,
+	incremental bool,
+	cbtParam CBTParam,
 	volMode uploader.PersistentVolumeMode,
 	uploaderCfg map[string]string,
 	updater uploader.ProgressUpdater) (int64, error) {
@@ -178,7 +180,7 @@ func (bp *blockProvider) RunRestore(

 	blkUploader := block.NewUploader(ctx, bp.bkRepo, updater, log)

-	size, err := blockRestoreFunc(ctx, blkUploader, bp.bkRepo, snapshotID, volumePath, uploaderCfg, log)
+	size, err := blockRestoreFunc(ctx, blkUploader, bp.bkRepo, snapshotID, volumePath, incremental, cbtParam.Source, cbtParam.Service, uploaderCfg, log)

 	// errors.Is, not ==: see the equivalent comment on the backup path above.
 	if errors.Is(err, block.ErrCanceled) {
@@ -412,7 +412,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) {
 	t.Run("restore", func(t *testing.T) {
 		orig := blockRestoreFunc
 		defer func() { blockRestoreFunc = orig }()
-		blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ map[string]string, _ logrus.FieldLogger) (int64, error) {
+		blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ bool, _ cbtservice.SourceInfo, _ cbtservice.Service, _ map[string]string, _ logrus.FieldLogger) (int64, error) {
 			return 0, errors.Wrap(block.ErrCanceled, "error restoring bdev")
 		}

@@ -422,7 +422,7 @@ func TestBlockProviderCancelThroughWrappedError(t *testing.T) {
 			log:           logrus.New(),
 		}

-		_, err := bp.RunRestore(t.Context(), "snap-1", "/dev/sda",
+		_, err := bp.RunRestore(t.Context(), "snap-1", "/dev/sda", false, CBTParam{},
 			uploader.PersistentVolumeBlock, map[string]string{}, &blockMockProgressUpdater{})

 		require.ErrorIs(t, err, ErrorCanceled)
@@ -496,9 +496,9 @@ func TestBlockProviderRunRestore(t *testing.T) {
 			var capturedSnapshotID string
 			var capturedVolumePath string

-			blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, snapshotID string, volumePath string, _ map[string]string, _ logrus.FieldLogger) (int64, error) {
+			blockRestoreFunc = func(ctx context.Context, blkUp block.Uploader, rep udmrepo.BackupRepo, snapshotID string, dest string, incremental bool, cbtSource cbtservice.SourceInfo, cbtService cbtservice.Service, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) {
 				capturedSnapshotID = snapshotID
-				capturedVolumePath = volumePath
+				capturedVolumePath = dest
 				return tc.mockRestoreSize, tc.mockRestoreErr
 			}

@@ -511,6 +511,8 @@ func TestBlockProviderRunRestore(t *testing.T) {
 				t.Context(),
 				tc.snapshotID,
 				tc.volumePath,
+				false,
+				CBTParam{},
 				uploader.PersistentVolumeBlock,
 				map[string]string{},
 				tc.updater,
@@ -211,6 +211,8 @@ func (kp *kopiaProvider) RunRestore(
 	ctx context.Context,
 	snapshotID string,
 	volumePath string,
+	incremental bool,
+	_ CBTParam,
 	volMode uploader.PersistentVolumeMode,
 	uploaderCfg map[string]string,
 	updater uploader.ProgressUpdater) (int64, error) {
@@ -234,7 +236,7 @@ func (kp *kopiaProvider) RunRestore(
 	// We use the cancel channel to control the restore cancel, so don't pass a context with cancel to Kopia restore.
 	// Otherwise, Kopia restore will not response to the cancel control but return an arbitrary error.
 	// Kopia restore cancel is not designed as well as Kopia backup which uses the context to control backup cancel all the way.
-	size, fileCount, err := kopiaRestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, volMode, uploaderCfg, log, restoreCancel)
+	size, fileCount, err := kopiaRestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, incremental, volMode, uploaderCfg, log, restoreCancel)

 	if err != nil {
 		return 0, errors.Wrapf(err, "Failed to run kopia restore")
@@ -119,20 +119,21 @@ func TestRunBackup(t *testing.T) {
 func TestRunRestore(t *testing.T) {
 	testCases := []struct {
 		name            string
-		hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error)
+		hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error)
 		notError        bool
 		volMode         uploader.PersistentVolumeMode
+		incremental     bool
 	}{
 		{
 			name: "normal restore",
-			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
+			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
 				return 0, 0, nil
 			},
 			notError: true,
 		},
 		{
 			name: "normal block mode restore",
-			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
+			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
 				return 0, 0, nil
 			},
 			volMode:  uploader.PersistentVolumeBlock,
@@ -140,7 +141,7 @@ func TestRunRestore(t *testing.T) {
 		},
 		{
 			name: "failed to restore",
-			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
+			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
 				return 0, 0, errors.New("failed to restore")
 			},
 			notError: false,
@@ -157,7 +158,7 @@ func TestRunRestore(t *testing.T) {
 				tc.volMode = uploader.PersistentVolumeFilesystem
 			}
 			kopiaRestoreFunc = tc.hookRestoreFunc
-			_, err := kp.RunRestore(t.Context(), "", "/var", tc.volMode, map[string]string{}, &updater)
+			_, err := kp.RunRestore(t.Context(), "", "/var", tc.incremental, CBTParam{}, tc.volMode, map[string]string{}, &updater)
 			if tc.notError {
 				assert.NoError(t, err)
 			} else {
@@ -223,8 +223,8 @@ func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, pa
 }

 // RunRestore provides a mock function for the type Provider
-func (_mock *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error) {
-	ret := _mock.Called(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
+func (_mock *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, incremental bool, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error) {
+	ret := _mock.Called(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)

 	if len(ret) == 0 {
 		panic("no return value specified for RunRestore")
@@ -232,16 +232,16 @@ func (_mock *Provider) RunRestore(ctx context.Context, snapshotID string, volume

 	var r0 int64
 	var r1 error
-	if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (int64, error)); ok {
-		return returnFunc(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
+	if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, bool, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (int64, error)); ok {
+		return returnFunc(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)
 	}
-	if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok {
-		r0 = returnFunc(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
+	if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, bool, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok {
+		r0 = returnFunc(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)
 	} else {
 		r0 = ret.Get(0).(int64)
 	}
-	if returnFunc, ok := ret.Get(1).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
-		r1 = returnFunc(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
+	if returnFunc, ok := ret.Get(1).(func(context.Context, string, string, bool, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
+		r1 = returnFunc(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)
 	} else {
 		r1 = ret.Error(1)
 	}
@@ -257,14 +257,16 @@ type Provider_RunRestore_Call struct {
 //   - ctx context.Context
 //   - snapshotID string
 //   - volumePath string
+//   - incremental bool
+//   - cbtParam provider.CBTParam
 //   - volMode uploader.PersistentVolumeMode
 //   - uploaderConfig map[string]string
 //   - updater uploader.ProgressUpdater
-func (_e *Provider_Expecter) RunRestore(ctx interface{}, snapshotID interface{}, volumePath interface{}, volMode interface{}, uploaderConfig interface{}, updater interface{}) *Provider_RunRestore_Call {
-	return &Provider_RunRestore_Call{Call: _e.mock.On("RunRestore", ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)}
+func (_e *Provider_Expecter) RunRestore(ctx interface{}, snapshotID interface{}, volumePath interface{}, incremental interface{}, cbtParam interface{}, volMode interface{}, uploaderConfig interface{}, updater interface{}) *Provider_RunRestore_Call {
+	return &Provider_RunRestore_Call{Call: _e.mock.On("RunRestore", ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)}
 }

-func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater)) *Provider_RunRestore_Call {
+func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID string, volumePath string, incremental bool, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater)) *Provider_RunRestore_Call {
 	_c.Call.Run(func(args mock.Arguments) {
 		var arg0 context.Context
 		if args[0] != nil {
@@ -278,17 +280,25 @@ func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID
 		if args[2] != nil {
 			arg2 = args[2].(string)
 		}
-		var arg3 uploader.PersistentVolumeMode
+		var arg3 bool
 		if args[3] != nil {
-			arg3 = args[3].(uploader.PersistentVolumeMode)
+			arg3 = args[3].(bool)
 		}
-		var arg4 map[string]string
+		var arg4 provider.CBTParam
 		if args[4] != nil {
-			arg4 = args[4].(map[string]string)
+			arg4 = args[4].(provider.CBTParam)
 		}
-		var arg5 uploader.ProgressUpdater
+		var arg5 uploader.PersistentVolumeMode
 		if args[5] != nil {
-			arg5 = args[5].(uploader.ProgressUpdater)
+			arg5 = args[5].(uploader.PersistentVolumeMode)
+		}
+		var arg6 map[string]string
+		if args[6] != nil {
+			arg6 = args[6].(map[string]string)
+		}
+		var arg7 uploader.ProgressUpdater
+		if args[7] != nil {
+			arg7 = args[7].(uploader.ProgressUpdater)
 		}
 		run(
 			arg0,
@@ -297,6 +307,8 @@ func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID
 			arg3,
 			arg4,
 			arg5,
+			arg6,
+			arg7,
 		)
 	})
 	return _c
@@ -307,7 +319,7 @@ func (_c *Provider_RunRestore_Call) Return(n int64, err error) *Provider_RunRest
 	return _c
 }

-func (_c *Provider_RunRestore_Call) RunAndReturn(run func(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error)) *Provider_RunRestore_Call {
+func (_c *Provider_RunRestore_Call) RunAndReturn(run func(ctx context.Context, snapshotID string, volumePath string, incremental bool, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error)) *Provider_RunRestore_Call {
 	_c.Call.Return(run)
 	return _c
 }
@@ -64,6 +64,8 @@ type Provider interface {
 		ctx context.Context,
 		snapshotID string,
 		volumePath string,
+		incremental bool,
+		cbtParam CBTParam,
 		volMode uploader.PersistentVolumeMode,
 		uploaderConfig map[string]string,
 		updater uploader.ProgressUpdater) (int64, error)
@@ -28,6 +28,7 @@ const (
 	ParallelFilesUpload = "ParallelFilesUpload"
 	WriteSparseFiles    = "WriteSparseFiles"
 	RestoreConcurrency  = "ParallelFilesDownload"
+	DeleteExtraFiles    = "DeleteExtraFiles"
 )

 func StoreBackupConfig(config *velerov1api.UploaderConfigForBackup) map[string]string {
@@ -47,6 +48,13 @@ func StoreRestoreConfig(config *velerov1api.UploaderConfigForRestore) map[string
 	if config.ParallelFilesDownload > 0 {
 		data[RestoreConcurrency] = strconv.Itoa(config.ParallelFilesDownload)
 	}
+
+	if config.DeleteExtraFiles != nil {
+		data[DeleteExtraFiles] = strconv.FormatBool(*config.DeleteExtraFiles)
+	} else {
+		data[DeleteExtraFiles] = strconv.FormatBool(false)
+	}
+
 	return data
 }

@@ -85,3 +93,15 @@ func GetRestoreConcurrency(uploaderCfg map[string]string) (int, error) {
 	}
 	return 0, nil
 }
+
+func GetDeleteExtraFiles(uploaderCfg map[string]string) (bool, error) {
+	deleteExtraFiles, ok := uploaderCfg[DeleteExtraFiles]
+	if ok {
+		deleteExtraFilesBool, err := strconv.ParseBool(deleteExtraFiles)
+		if err != nil {
+			return false, errors.Wrap(err, "failed to parse DeleteExtraFiles config")
+		}
+		return deleteExtraFilesBool, nil
+	}
+	return false, nil
+}
@@ -58,6 +58,7 @@ func TestStoreRestoreConfig(t *testing.T) {
 			},
 			expectedData: map[string]string{
 				WriteSparseFiles: "true",
+				DeleteExtraFiles: "false",
 			},
 		},
 		{
@@ -67,6 +68,7 @@ func TestStoreRestoreConfig(t *testing.T) {
 			},
 			expectedData: map[string]string{
 				WriteSparseFiles: "false",
+				DeleteExtraFiles: "false",
 			},
 		},
 		{
@@ -76,6 +78,7 @@ func TestStoreRestoreConfig(t *testing.T) {
 			},
 			expectedData: map[string]string{
 				WriteSparseFiles: "false", // Assuming default value is false for nil case
+				DeleteExtraFiles: "false",
 			},
 		},
 		{
@@ -86,6 +89,37 @@ func TestStoreRestoreConfig(t *testing.T) {
 			expectedData: map[string]string{
 				RestoreConcurrency: "5",
 				WriteSparseFiles:   "false",
+				DeleteExtraFiles:   "false",
+			},
+		},
+		{
+			name: "DeleteExtraFiles is true",
+			config: &velerov1api.UploaderConfigForRestore{
+				DeleteExtraFiles: &boolTrue,
+			},
+			expectedData: map[string]string{
+				WriteSparseFiles: "false",
+				DeleteExtraFiles: "true",
+			},
+		},
+		{
+			name: "DeleteExtraFiles is false",
+			config: &velerov1api.UploaderConfigForRestore{
+				DeleteExtraFiles: &boolFalse,
+			},
+			expectedData: map[string]string{
+				WriteSparseFiles: "false",
+				DeleteExtraFiles: "false",
+			},
+		},
+		{
+			name: "DeleteExtraFiles is nil",
+			config: &velerov1api.UploaderConfigForRestore{
+				DeleteExtraFiles: nil,
+			},
+			expectedData: map[string]string{
+				WriteSparseFiles: "false",
+				DeleteExtraFiles: "false", // Assuming default value is false for nil case
 			},
 		},
 	}
@@ -240,3 +274,51 @@ func TestGetRestoreConcurrency(t *testing.T) {
 		})
 	}
 }
+
+func TestGetDeleteExtraFiles(t *testing.T) {
+	tests := []struct {
+		name           string
+		uploaderCfg    map[string]string
+		expectedResult bool
+		expectedError  error
+	}{
+		{
+			name:           "Valid DeleteExtraFiles (true)",
+			uploaderCfg:    map[string]string{DeleteExtraFiles: "true"},
+			expectedResult: true,
+			expectedError:  nil,
+		},
+		{
+			name:           "Valid DeleteExtraFiles (false)",
+			uploaderCfg:    map[string]string{DeleteExtraFiles: "false"},
+			expectedResult: false,
+			expectedError:  nil,
+		},
+		{
+			name:           "Invalid DeleteExtraFiles (not a boolean)",
+			uploaderCfg:    map[string]string{DeleteExtraFiles: "invalid"},
+			expectedResult: false,
+			expectedError:  errors.Wrap(errors.New("strconv.ParseBool: parsing \"invalid\": invalid syntax"), "failed to parse DeleteExtraFiles config"),
+		},
+		{
+			name:           "Missing DeleteExtraFiles",
+			uploaderCfg:    map[string]string{},
+			expectedResult: false,
+			expectedError:  nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			result, err := GetDeleteExtraFiles(test.uploaderCfg)
+
+			if result != test.expectedResult {
+				t.Errorf("Expected result %t, but got %t", test.expectedResult, result)
+			}
+
+			if (err == nil && test.expectedError != nil) || (err != nil && test.expectedError == nil) || (err != nil && test.expectedError != nil && err.Error() != test.expectedError.Error()) {
+				t.Errorf("Expected error '%v', but got '%v'", test.expectedError, err)
+			}
+		})
+	}
+}