Merge branch 'main' of https://github.com/qiuming-best/velero into support-restore-sparse

changelogs/unreleased/6958-blackpiglet

@@ -0,0 +1 @@
+Change controller-runtime List option from MatchingFields to ListOptions

changelogs/unreleased/7069-27149chen

@@ -0,0 +1 @@
+improve discoveryHelper.Refresh() in restore

changelogs/unreleased/7117-allenxu404

@@ -0,0 +1 @@
+Add hooks status to backup/restore CR

changelogs/unreleased/7130-qiuming-best

@@ -0,0 +1 @@
+Node agent restart enhancement

config/crd/v1/bases/velero.io_backups.yaml

@@ -35,14 +35,6 @@ spec:
           spec:
             description: BackupSpec defines the specification for a Velero backup.
             properties:
-              backupConfig:
-                description: BackupConfig defines the configuration for the backup.
-                properties:
-                  parallelFilesUpload:
-                    description: ParallelFilesUpload is the number of files parallel
-                      uploads to perform when using the uploader.
-                    type: integer
-                type: object
               csiSnapshotTimeout:
                 description: CSISnapshotTimeout specifies the time used to wait for
                   CSI VolumeSnapshot status turns to ReadyToUse during creation, before
@@ -485,6 +477,14 @@ spec:
                 description: TTL is a time.Duration-parseable string describing how
                   long the Backup should be retained for.
                 type: string
+              uploaderConfig:
+                description: UploaderConfig specifies the configuration for the uploader.
+                properties:
+                  parallelFilesUpload:
+                    description: ParallelFilesUpload is the number of files parallel
+                      uploads to perform when using the uploader.
+                    type: integer
+                type: object
               volumeSnapshotLocations:
                 description: VolumeSnapshotLocations is a list containing names of
                   VolumeSnapshotLocations associated with this backup.
@@ -543,6 +543,22 @@ spec:
                 description: FormatVersion is the backup format version, including
                   major, minor, and patch version.
                 type: string
+              hookStatus:
+                description: HookStatus contains information about the status of the
+                  hooks.
+                nullable: true
+                properties:
+                  hooksAttempted:
+                    description: HooksAttempted is the total number of attempted hooks
+                      Specifically, HooksAttempted represents the number of hooks
+                      that failed to execute and the number of hooks that executed
+                      successfully.
+                    type: integer
+                  hooksFailed:
+                    description: HooksFailed is the total number of hooks which ended
+                      with an error
+                    type: integer
+                type: object
               phase:
                 description: Phase is the current state of the Backup.
                 enum:

config/crd/v1/bases/velero.io_restores.yaml

@@ -387,14 +387,6 @@ spec:
                 - name
                 type: object
                 x-kubernetes-map-type: atomic
-              restoreConfig:
-                description: RestoreConfig specifies the configuration for the restore.
-                properties:
-                  writeSparseFiles:
-                    description: WriteSparseFiles is a flag to indicate whether write
-                      files sparsely or not.
-                    type: boolean
-                type: object
               restorePVs:
                 description: RestorePVs specifies whether to restore all included
                   PVs from snapshot
@@ -426,6 +418,14 @@ spec:
                   restore from the most recent successful backup created from this
                   schedule.
                 type: string
+              uploaderConfig:
+                description: UploaderConfig specifies the configuration for the restore.
+                properties:
+                  writeSparseFiles:
+                    description: WriteSparseFiles is a flag to indicate whether write
+                      files sparsely or not.
+                    type: boolean
+                type: object
             required:
             - backupName
             type: object
@@ -448,6 +448,22 @@ spec:
                 description: FailureReason is an error that caused the entire restore
                   to fail.
                 type: string
+              hookStatus:
+                description: HookStatus contains information about the status of the
+                  hooks.
+                nullable: true
+                properties:
+                  hooksAttempted:
+                    description: HooksAttempted is the total number of attempted hooks
+                      Specifically, HooksAttempted represents the number of hooks
+                      that failed to execute and the number of hooks that executed
+                      successfully.
+                    type: integer
+                  hooksFailed:
+                    description: HooksFailed is the total number of hooks which ended
+                      with an error
+                    type: integer
+                type: object
               phase:
                 description: Phase is the current state of the Restore
                 enum:

config/crd/v1/bases/velero.io_schedules.yaml

@@ -65,14 +65,6 @@ spec:
                 description: Template is the definition of the Backup to be run on
                   the provided schedule
                 properties:
-                  backupConfig:
-                    description: BackupConfig defines the configuration for the backup.
-                    properties:
-                      parallelFilesUpload:
-                        description: ParallelFilesUpload is the number of files parallel
-                          uploads to perform when using the uploader.
-                        type: integer
-                    type: object
                   csiSnapshotTimeout:
                     description: CSISnapshotTimeout specifies the time used to wait
                       for CSI VolumeSnapshot status turns to ReadyToUse during creation,
@@ -522,6 +514,15 @@ spec:
                     description: TTL is a time.Duration-parseable string describing
                       how long the Backup should be retained for.
                     type: string
+                  uploaderConfig:
+                    description: UploaderConfig specifies the configuration for the
+                      uploader.
+                    properties:
+                      parallelFilesUpload:
+                        description: ParallelFilesUpload is the number of files parallel
+                          uploads to perform when using the uploader.
+                        type: integer
+                    type: object
                   volumeSnapshotLocations:
                     description: VolumeSnapshotLocations is a list containing names
                       of VolumeSnapshotLocations associated with this backup.

design/velero-uploader-configuration.md

@@ -25,7 +25,7 @@ type UploaderConfig struct {
 }
 ```
 
-### Integration with Backup & Restore CRD
+### Integration with Backup CRD
 The Velero CLI will support an uploader configuration-related flag, allowing users to set the value when creating backups or restores. This value will be stored in the `UploaderConfig` field within the `Backup` CRD and `Restore` CRD:
 
 ```go
@@ -122,44 +122,5 @@ Roughly, the process is as follows:
 3. Each respective controller within the CRs calls the uploader, and the ParallelFilesUpload from UploaderConfig in CRs is passed to the uploader. 
 4. When the uploader subsequently calls the Kopia API, it can use the ParallelFilesUpload to set the MaxParallelFileReads parameter, and if the uploader calls the Restic command it would output one warning log for Restic does not support this feature.
 
-### Sparse Option For Kopia & Restic Restore
-In many system files, there are numerous zero bytes or empty blocks that still occupy physical storage space. Sparse backup employs a more intelligent approach by only backing up the actual data-containing portions. For those empty blocks or zero bytes, it merely records their presence without actually storing them. This can significantly reduce the storage space required for backups, especially in situations where there is a substantial amount of empty data in large file systems.
-
-Below are the key steps that should be added to support this new feature.
-#### Velero CLI 
-The Velero CLI will support a `--write-sparse-files` flag, allowing users to set the `WriteSparseFiles` value when creating restores with Restic or Kopia uploader.
-
-#### UploaderConfig 
-below the sub-option `WriteSparseFiles` is added into UploaderConfig:
-
-```go
-type UploaderConfig struct {
-    // +optional
-	WriteSparseFiles bool `json:"writeSparseFiles,omitempty"`
-}
-```
-
-### Enable Sparse in Restic
-For Restic, it could be enabled by pass the flag `--sparse` in creating restore:
-
-```bash
-restic restore create --sparse $snapshotID
-```
-
-### Enable Sparse in Kopia
-For Kopia, it could be enabled this feature by the `WriteSparseFiles` field in the [FilesystemOutput](https://pkg.go.dev/github.com/kopia/kopia@v0.13.0/snapshot/restore#FilesystemOutput).
-
-```golang
-fsOutput := &restore.FilesystemOutput{
-		WriteSparseFiles:       veleroCfg.WriteSparseFiles,
-	}
-```
-
-Roughly, the process is as follows: 
-1. Users pass the WriteSparseFiles parameter and its value through the Velero CLI. This parameter and its value are stored as a sub-option within UploaderConfig and then placed into the Restore CR. 
-2. When users perform file system restores, UploaderConfig is passed to the PodVolumeRestore CR. When users use the Data-mover for restores, it is passed to the DataDownload CR. 
-3. Each respective controller within the CRs calls the uploader, and the WriteSparseFiles from UploaderConfig in CRs is passed to the uploader. 
-4. When the uploader subsequently calls the Kopia API, it can use the WriteSparseFiles to set the WriteSparseFiles parameter, and if the uploader calls the Restic command it would append `--sparse` flag within the restore command.
-
 ## Alternatives Considered
 To enhance extensibility further, the option of storing `UploaderConfig` in a Kubernetes ConfigMap can be explored, this approach would allow the addition and modification of configuration options without the need to modify the CRD.

internal/hook/hook_tracker.go

@@ -0,0 +1,137 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hook
+
+import "sync"
+
+const (
+	HookSourceAnnotation = "annotation"
+	HookSourceSpec       = "spec"
+)
+
+// hookTrackerKey identifies a backup/restore hook
+type hookTrackerKey struct {
+	// PodNamespace indicates the namespace of pod where hooks are executed.
+	// For hooks specified in the backup/restore spec, this field is the namespace of an applicable pod.
+	// For hooks specified in pod annotation, this field is the namespace of pod where hooks are annotated.
+	podNamespace string
+	// PodName indicates the pod where hooks are executed.
+	// For hooks specified in the backup/restore spec, this field is an applicable pod name.
+	// For hooks specified in pod annotation, this field is the pod where hooks are annotated.
+	podName string
+	// HookPhase is only for backup hooks, for restore hooks, this field is empty.
+	hookPhase hookPhase
+	// HookName is only for hooks specified in the backup/restore spec.
+	// For hooks specified in pod annotation, this field is empty or "<from-annotation>".
+	hookName string
+	// HookSource indicates where hooks come from.
+	hookSource string
+	// Container indicates the container hooks use.
+	// For hooks specified in the backup/restore spec, the container might be the same under different hookName.
+	container string
+}
+
+// hookTrackerVal records the execution status of a specific hook.
+// hookTrackerVal is extensible to accommodate additional fields as needs develop.
+type hookTrackerVal struct {
+	// HookFailed indicates if hook failed to execute.
+	hookFailed bool
+	// hookExecuted indicates if hook already execute.
+	hookExecuted bool
+}
+
+// HookTracker tracks all hooks' execution status
+type HookTracker struct {
+	lock    *sync.RWMutex
+	tracker map[hookTrackerKey]hookTrackerVal
+}
+
+// NewHookTracker creates a hookTracker.
+func NewHookTracker() *HookTracker {
+	return &HookTracker{
+		lock:    &sync.RWMutex{},
+		tracker: make(map[hookTrackerKey]hookTrackerVal),
+	}
+}
+
+// Add adds a hook to the tracker
+func (ht *HookTracker) Add(podNamespace, podName, container, source, hookName string, hookPhase hookPhase) {
+	ht.lock.Lock()
+	defer ht.lock.Unlock()
+
+	key := hookTrackerKey{
+		podNamespace: podNamespace,
+		podName:      podName,
+		hookSource:   source,
+		container:    container,
+		hookPhase:    hookPhase,
+		hookName:     hookName,
+	}
+
+	if _, ok := ht.tracker[key]; !ok {
+		ht.tracker[key] = hookTrackerVal{
+			hookFailed:   false,
+			hookExecuted: false,
+		}
+	}
+}
+
+// Record records the hook's execution status
+func (ht *HookTracker) Record(podNamespace, podName, container, source, hookName string, hookPhase hookPhase, hookFailed bool) {
+	ht.lock.Lock()
+	defer ht.lock.Unlock()
+
+	key := hookTrackerKey{
+		podNamespace: podNamespace,
+		podName:      podName,
+		hookSource:   source,
+		container:    container,
+		hookPhase:    hookPhase,
+		hookName:     hookName,
+	}
+
+	if _, ok := ht.tracker[key]; ok {
+		ht.tracker[key] = hookTrackerVal{
+			hookFailed:   hookFailed,
+			hookExecuted: true,
+		}
+	}
+}
+
+// Stat calculates the number of attempted hooks and failed hooks
+func (ht *HookTracker) Stat() (hookAttemptedCnt int, hookFailed int) {
+	ht.lock.RLock()
+	defer ht.lock.RUnlock()
+
+	for _, hookInfo := range ht.tracker {
+		if hookInfo.hookExecuted {
+			hookAttemptedCnt++
+			if hookInfo.hookFailed {
+				hookFailed++
+			}
+		}
+	}
+	return
+}
+
+// GetTracker gets the tracker inside HookTracker
+func (ht *HookTracker) GetTracker() map[hookTrackerKey]hookTrackerVal {
+	ht.lock.RLock()
+	defer ht.lock.RUnlock()
+
+	return ht.tracker
+}

internal/hook/hook_tracker_test.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hook
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewHookTracker(t *testing.T) {
+	tracker := NewHookTracker()
+
+	assert.NotNil(t, tracker)
+	assert.Empty(t, tracker.tracker)
+}
+
+func TestHookTracker_Add(t *testing.T) {
+	tracker := NewHookTracker()
+
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre)
+
+	key := hookTrackerKey{
+		podNamespace: "ns1",
+		podName:      "pod1",
+		container:    "container1",
+		hookPhase:    PhasePre,
+		hookSource:   HookSourceAnnotation,
+		hookName:     "h1",
+	}
+
+	_, ok := tracker.tracker[key]
+	assert.True(t, ok)
+}
+
+func TestHookTracker_Record(t *testing.T) {
+	tracker := NewHookTracker()
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre)
+	tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, true)
+
+	key := hookTrackerKey{
+		podNamespace: "ns1",
+		podName:      "pod1",
+		container:    "container1",
+		hookPhase:    PhasePre,
+		hookSource:   HookSourceAnnotation,
+		hookName:     "h1",
+	}
+
+	info := tracker.tracker[key]
+	assert.True(t, info.hookFailed)
+}
+
+func TestHookTracker_Stat(t *testing.T) {
+	tracker := NewHookTracker()
+
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre)
+	tracker.Add("ns2", "pod2", "container1", HookSourceAnnotation, "h2", PhasePre)
+	tracker.Record("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre, true)
+
+	attempted, failed := tracker.Stat()
+	assert.Equal(t, 1, attempted)
+	assert.Equal(t, 1, failed)
+}
+
+func TestHookTracker_Get(t *testing.T) {
+	tracker := NewHookTracker()
+	tracker.Add("ns1", "pod1", "container1", HookSourceAnnotation, "h1", PhasePre)
+
+	tr := tracker.GetTracker()
+	assert.NotNil(t, tr)
+
+	t.Logf("tracker :%+v", tr)
+}

internal/hook/item_hook_handler.go

@@ -82,6 +82,7 @@ type ItemHookHandler interface {
 		obj runtime.Unstructured,
 		resourceHooks []ResourceHook,
 		phase hookPhase,
+		hookTracker *HookTracker,
 	) error
 }
 
@@ -200,6 +201,7 @@ func (h *DefaultItemHookHandler) HandleHooks(
 	obj runtime.Unstructured,
 	resourceHooks []ResourceHook,
 	phase hookPhase,
+	hookTracker *HookTracker,
 ) error {
 	// We only support hooks on pods right now
 	if groupResource != kuberesource.Pods {
@@ -221,15 +223,21 @@ func (h *DefaultItemHookHandler) HandleHooks(
 		hookFromAnnotations = getPodExecHookFromAnnotations(metadata.GetAnnotations(), "", log)
 	}
 	if hookFromAnnotations != nil {
+		hookTracker.Add(namespace, name, hookFromAnnotations.Container, HookSourceAnnotation, "", phase)
+
 		hookLog := log.WithFields(
 			logrus.Fields{
-				"hookSource": "annotation",
+				"hookSource": HookSourceAnnotation,
 				"hookType":   "exec",
 				"hookPhase":  phase,
 			},
 		)
+
+		hookTracker.Record(namespace, name, hookFromAnnotations.Container, HookSourceAnnotation, "", phase, false)
 		if err := h.PodCommandExecutor.ExecutePodCommand(hookLog, obj.UnstructuredContent(), namespace, name, "<from-annotation>", hookFromAnnotations); err != nil {
 			hookLog.WithError(err).Error("Error executing hook")
+			hookTracker.Record(namespace, name, hookFromAnnotations.Container, HookSourceAnnotation, "", phase, true)
+
 			if hookFromAnnotations.OnError == velerov1api.HookErrorModeFail {
 				return err
 			}
@@ -240,6 +248,8 @@ func (h *DefaultItemHookHandler) HandleHooks(
 
 	labels := labels.Set(metadata.GetLabels())
 	// Otherwise, check for hooks defined in the backup spec.
+	// modeFailError records the error from the hook with "Fail" error mode
+	var modeFailError error
 	for _, resourceHook := range resourceHooks {
 		if !resourceHook.Selector.applicableTo(groupResource, namespace, labels) {
 			continue
@@ -251,21 +261,30 @@ func (h *DefaultItemHookHandler) HandleHooks(
 		} else {
 			hooks = resourceHook.Post
 		}
+
 		for _, hook := range hooks {
 			if groupResource == kuberesource.Pods {
 				if hook.Exec != nil {
-					hookLog := log.WithFields(
-						logrus.Fields{
-							"hookSource": "backupSpec",
-							"hookType":   "exec",
-							"hookPhase":  phase,
-						},
-					)
-					err := h.PodCommandExecutor.ExecutePodCommand(hookLog, obj.UnstructuredContent(), namespace, name, resourceHook.Name, hook.Exec)
-					if err != nil {
-						hookLog.WithError(err).Error("Error executing hook")
-						if hook.Exec.OnError == velerov1api.HookErrorModeFail {
-							return err
+					hookTracker.Add(namespace, name, hook.Exec.Container, HookSourceSpec, resourceHook.Name, phase)
+					// The remaining hooks will only be executed if modeFailError is nil.
+					// Otherwise, execution will stop and only hook collection will occur.
+					if modeFailError == nil {
+						hookLog := log.WithFields(
+							logrus.Fields{
+								"hookSource": HookSourceSpec,
+								"hookType":   "exec",
+								"hookPhase":  phase,
+							},
+						)
+
+						hookTracker.Record(namespace, name, hook.Exec.Container, HookSourceSpec, resourceHook.Name, phase, false)
+						err := h.PodCommandExecutor.ExecutePodCommand(hookLog, obj.UnstructuredContent(), namespace, name, resourceHook.Name, hook.Exec)
+						if err != nil {
+							hookLog.WithError(err).Error("Error executing hook")
+							hookTracker.Record(namespace, name, hook.Exec.Container, HookSourceSpec, resourceHook.Name, phase, true)
+							if hook.Exec.OnError == velerov1api.HookErrorModeFail {
+								modeFailError = err
+							}
 						}
 					}
 				}
@@ -273,7 +292,7 @@ func (h *DefaultItemHookHandler) HandleHooks(
 		}
 	}
 
-	return nil
+	return modeFailError
 }
 
 // NoOpItemHookHandler is the an itemHookHandler for the Finalize controller where hooks don't run
@@ -285,6 +304,7 @@ func (h *NoOpItemHookHandler) HandleHooks(
 	obj runtime.Unstructured,
 	resourceHooks []ResourceHook,
 	phase hookPhase,
+	hookTracker *HookTracker,
 ) error {
 	return nil
 }
@@ -514,6 +534,7 @@ func GroupRestoreExecHooks(
 	resourceRestoreHooks []ResourceRestoreHook,
 	pod *corev1api.Pod,
 	log logrus.FieldLogger,
+	hookTrack *HookTracker,
 ) (map[string][]PodExecRestoreHook, error) {
 	byContainer := map[string][]PodExecRestoreHook{}
 
@@ -530,10 +551,11 @@ func GroupRestoreExecHooks(
 		if hookFromAnnotation.Container == "" {
 			hookFromAnnotation.Container = pod.Spec.Containers[0].Name
 		}
+		hookTrack.Add(metadata.GetNamespace(), metadata.GetName(), hookFromAnnotation.Container, HookSourceAnnotation, "<from-annotation>", hookPhase(""))
 		byContainer[hookFromAnnotation.Container] = []PodExecRestoreHook{
 			{
 				HookName:   "<from-annotation>",
-				HookSource: "annotation",
+				HookSource: HookSourceAnnotation,
 				Hook:       *hookFromAnnotation,
 			},
 		}
@@ -554,7 +576,7 @@ func GroupRestoreExecHooks(
 			named := PodExecRestoreHook{
 				HookName:   rrh.Name,
 				Hook:       *rh.Exec,
-				HookSource: "backupSpec",
+				HookSource: HookSourceSpec,
 			}
 			// default to false if attr WaitForReady not set
 			if named.Hook.WaitForReady == nil {
@@ -564,6 +586,7 @@ func GroupRestoreExecHooks(
 			if named.Hook.Container == "" {
 				named.Hook.Container = pod.Spec.Containers[0].Name
 			}
+			hookTrack.Add(metadata.GetNamespace(), metadata.GetName(), named.Hook.Container, HookSourceSpec, rrh.Name, hookPhase(""))
 			byContainer[named.Hook.Container] = append(byContainer[named.Hook.Container], named)
 		}
 	}

internal/hook/item_hook_handler_test.go

@@ -108,6 +108,7 @@ func TestHandleHooksSkips(t *testing.T) {
 		},
 	}
 
+	hookTracker := NewHookTracker()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			podCommandExecutor := &velerotest.MockPodCommandExecutor{}
@@ -118,7 +119,7 @@ func TestHandleHooksSkips(t *testing.T) {
 			}
 
 			groupResource := schema.ParseGroupResource(test.groupResource)
-			err := h.HandleHooks(velerotest.NewLogger(), groupResource, test.item, test.hooks, PhasePre)
+			err := h.HandleHooks(velerotest.NewLogger(), groupResource, test.item, test.hooks, PhasePre, hookTracker)
 			assert.NoError(t, err)
 		})
 	}
@@ -485,7 +486,8 @@ func TestHandleHooks(t *testing.T) {
 			}
 
 			groupResource := schema.ParseGroupResource(test.groupResource)
-			err := h.HandleHooks(velerotest.NewLogger(), groupResource, test.item, test.hooks, test.phase)
+			hookTracker := NewHookTracker()
+			err := h.HandleHooks(velerotest.NewLogger(), groupResource, test.item, test.hooks, test.phase, hookTracker)
 
 			if test.expectedError != nil {
 				assert.EqualError(t, err, test.expectedError.Error())
@@ -861,7 +863,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "<from-annotation>",
-						HookSource: "annotation",
+						HookSource: HookSourceAnnotation,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:    "container1",
 							Command:      []string{"/usr/bin/foo"},
@@ -892,7 +894,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "<from-annotation>",
-						HookSource: "annotation",
+						HookSource: HookSourceAnnotation,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:    "container1",
 							Command:      []string{"/usr/bin/foo"},
@@ -933,7 +935,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "hook1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:    "container1",
 							Command:      []string{"/usr/bin/foo"},
@@ -973,7 +975,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "hook1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:    "container1",
 							Command:      []string{"/usr/bin/foo"},
@@ -1021,7 +1023,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "<from-annotation>",
-						HookSource: "annotation",
+						HookSource: HookSourceAnnotation,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:    "container1",
 							Command:      []string{"/usr/bin/foo"},
@@ -1140,7 +1142,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "hook1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:    "container1",
 							Command:      []string{"/usr/bin/foo"},
@@ -1152,7 +1154,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 					},
 					{
 						HookName:   "hook1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:    "container1",
 							Command:      []string{"/usr/bin/bar"},
@@ -1164,7 +1166,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 					},
 					{
 						HookName:   "hook2",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:    "container1",
 							Command:      []string{"/usr/bin/aaa"},
@@ -1178,7 +1180,7 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 				"container2": {
 					{
 						HookName:   "hook1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:    "container2",
 							Command:      []string{"/usr/bin/baz"},
@@ -1192,9 +1194,11 @@ func TestGroupRestoreExecHooks(t *testing.T) {
 			},
 		},
 	}
+
+	hookTracker := NewHookTracker()
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			actual, err := GroupRestoreExecHooks(tc.resourceRestoreHooks, tc.pod, velerotest.NewLogger())
+			actual, err := GroupRestoreExecHooks(tc.resourceRestoreHooks, tc.pod, velerotest.NewLogger(), hookTracker)
 			assert.Nil(t, err)
 			assert.Equal(t, tc.expected, actual)
 		})
@@ -1983,3 +1987,494 @@ func TestValidateContainer(t *testing.T) {
 	// noCommand string should return expected error as result.
 	assert.Equal(t, expectedError, ValidateContainer([]byte(noCommand)))
 }
+
+func TestBackupHookTracker(t *testing.T) {
+	type podWithHook struct {
+		item                  runtime.Unstructured
+		hooks                 []ResourceHook
+		hookErrorsByContainer map[string]error
+		expectedPodHook       *velerov1api.ExecHook
+		expectedPodHookError  error
+		expectedError         error
+	}
+	test1 := []struct {
+		name                  string
+		phase                 hookPhase
+		groupResource         string
+		pods                  []podWithHook
+		hookTracker           *HookTracker
+		expectedHookAttempted int
+		expectedHookFailed    int
+	}{
+		{
+			name:                  "a pod with spec hooks, no error",
+			phase:                 PhasePre,
+			groupResource:         "pods",
+			hookTracker:           NewHookTracker(),
+			expectedHookAttempted: 2,
+			expectedHookFailed:    0,
+			pods: []podWithHook{
+				{
+					item: velerotest.UnstructuredOrDie(`
+					{
+						"apiVersion": "v1",
+						"kind": "Pod",
+						"metadata": {
+							"namespace": "ns",
+							"name": "name"
+						}
+					}`),
+					hooks: []ResourceHook{
+						{
+							Name: "hook1",
+							Pre: []velerov1api.BackupResourceHook{
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1a",
+										Command:   []string{"pre-1a"},
+									},
+								},
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1b",
+										Command:   []string{"pre-1b"},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:                  "a pod with spec hooks and same container under different hook name, no error",
+			phase:                 PhasePre,
+			groupResource:         "pods",
+			hookTracker:           NewHookTracker(),
+			expectedHookAttempted: 4,
+			expectedHookFailed:    0,
+			pods: []podWithHook{
+				{
+					item: velerotest.UnstructuredOrDie(`
+					{
+						"apiVersion": "v1",
+						"kind": "Pod",
+						"metadata": {
+							"namespace": "ns",
+							"name": "name"
+						}
+					}`),
+					hooks: []ResourceHook{
+						{
+							Name: "hook1",
+							Pre: []velerov1api.BackupResourceHook{
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1a",
+										Command:   []string{"pre-1a"},
+									},
+								},
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1b",
+										Command:   []string{"pre-1b"},
+									},
+								},
+							},
+						},
+						{
+							Name: "hook2",
+							Pre: []velerov1api.BackupResourceHook{
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1a",
+										Command:   []string{"2a"},
+									},
+								},
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "2b",
+										Command:   []string{"2b"},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:                  "a pod with spec hooks, on error=fail",
+			phase:                 PhasePre,
+			groupResource:         "pods",
+			hookTracker:           NewHookTracker(),
+			expectedHookAttempted: 3,
+			expectedHookFailed:    2,
+			pods: []podWithHook{
+				{
+					item: velerotest.UnstructuredOrDie(`
+					{
+						"apiVersion": "v1",
+						"kind": "Pod",
+						"metadata": {
+							"namespace": "ns",
+							"name": "name"
+						}
+					}`),
+					hooks: []ResourceHook{
+						{
+							Name: "hook1",
+							Pre: []velerov1api.BackupResourceHook{
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1a",
+										Command:   []string{"1a"},
+										OnError:   velerov1api.HookErrorModeContinue,
+									},
+								},
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1b",
+										Command:   []string{"1b"},
+									},
+								},
+							},
+						},
+						{
+							Name: "hook2",
+							Pre: []velerov1api.BackupResourceHook{
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "2",
+										Command:   []string{"2"},
+										OnError:   velerov1api.HookErrorModeFail,
+									},
+								},
+							},
+						},
+						{
+							Name: "hook3",
+							Pre: []velerov1api.BackupResourceHook{
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "3",
+										Command:   []string{"3"},
+									},
+								},
+							},
+						},
+					},
+					hookErrorsByContainer: map[string]error{
+						"1a": errors.New("1a error, but continue"),
+						"2":  errors.New("2 error, fail"),
+					},
+				},
+			},
+		},
+		{
+			name:                  "a pod with annotation and spec hooks",
+			phase:                 PhasePre,
+			groupResource:         "pods",
+			hookTracker:           NewHookTracker(),
+			expectedHookAttempted: 1,
+			expectedHookFailed:    0,
+			pods: []podWithHook{
+				{
+					item: velerotest.UnstructuredOrDie(`
+					{
+						"apiVersion": "v1",
+						"kind": "Pod",
+						"metadata": {
+							"namespace": "ns",
+							"name": "name",
+							"annotations": {
+								"hook.backup.velero.io/container": "c",
+								"hook.backup.velero.io/command": "/bin/ls"
+							}
+						}
+					}`),
+					expectedPodHook: &velerov1api.ExecHook{
+						Container: "c",
+						Command:   []string{"/bin/ls"},
+					},
+					hooks: []ResourceHook{
+						{
+							Name: "hook1",
+							Pre: []velerov1api.BackupResourceHook{
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1a",
+										Command:   []string{"1a"},
+										OnError:   velerov1api.HookErrorModeContinue,
+									},
+								},
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1b",
+										Command:   []string{"1b"},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:                  "a pod with annotation, on error=fail",
+			phase:                 PhasePre,
+			groupResource:         "pods",
+			hookTracker:           NewHookTracker(),
+			expectedHookAttempted: 1,
+			expectedHookFailed:    1,
+			pods: []podWithHook{
+				{
+					item: velerotest.UnstructuredOrDie(`
+					{
+						"apiVersion": "v1",
+						"kind": "Pod",
+						"metadata": {
+							"namespace": "ns",
+							"name": "name",
+							"annotations": {
+								"hook.backup.velero.io/container": "c",
+								"hook.backup.velero.io/command": "/bin/ls",
+								"hook.backup.velero.io/on-error": "Fail"
+							}
+						}
+					}`),
+					expectedPodHook: &velerov1api.ExecHook{
+						Container: "c",
+						Command:   []string{"/bin/ls"},
+						OnError:   velerov1api.HookErrorModeFail,
+					},
+					expectedPodHookError: errors.New("pod hook error"),
+				},
+			},
+		},
+		{
+			name:                  "two pods, one with annotation, the other with spec",
+			phase:                 PhasePre,
+			groupResource:         "pods",
+			hookTracker:           NewHookTracker(),
+			expectedHookAttempted: 3,
+			expectedHookFailed:    1,
+			pods: []podWithHook{
+				{
+					item: velerotest.UnstructuredOrDie(`
+					{
+						"apiVersion": "v1",
+						"kind": "Pod",
+						"metadata": {
+							"namespace": "ns",
+							"name": "name",
+							"annotations": {
+								"hook.backup.velero.io/container": "c",
+								"hook.backup.velero.io/command": "/bin/ls",
+								"hook.backup.velero.io/on-error": "Fail"
+							}
+						}
+					}`),
+					expectedPodHook: &velerov1api.ExecHook{
+						Container: "c",
+						Command:   []string{"/bin/ls"},
+						OnError:   velerov1api.HookErrorModeFail,
+					},
+					expectedPodHookError: errors.New("pod hook error"),
+				},
+				{
+					item: velerotest.UnstructuredOrDie(`
+					{
+						"apiVersion": "v1",
+						"kind": "Pod",
+						"metadata": {
+							"namespace": "ns",
+							"name": "name"
+						}
+					}`),
+					hooks: []ResourceHook{
+						{
+							Name: "hook1",
+							Pre: []velerov1api.BackupResourceHook{
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1a",
+										Command:   []string{"pre-1a"},
+									},
+								},
+								{
+									Exec: &velerov1api.ExecHook{
+										Container: "1b",
+										Command:   []string{"pre-1b"},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range test1 {
+		t.Run(test.name, func(t *testing.T) {
+			podCommandExecutor := &velerotest.MockPodCommandExecutor{}
+			defer podCommandExecutor.AssertExpectations(t)
+
+			h := &DefaultItemHookHandler{
+				PodCommandExecutor: podCommandExecutor,
+			}
+
+			groupResource := schema.ParseGroupResource(test.groupResource)
+			hookTracker := test.hookTracker
+
+			for _, pod := range test.pods {
+				if pod.expectedPodHook != nil {
+					podCommandExecutor.On("ExecutePodCommand", mock.Anything, pod.item.UnstructuredContent(), "ns", "name", "<from-annotation>", pod.expectedPodHook).Return(pod.expectedPodHookError)
+				} else {
+				hookLoop:
+					for _, resourceHook := range pod.hooks {
+						for _, hook := range resourceHook.Pre {
+							hookError := pod.hookErrorsByContainer[hook.Exec.Container]
+							podCommandExecutor.On("ExecutePodCommand", mock.Anything, pod.item.UnstructuredContent(), "ns", "name", resourceHook.Name, hook.Exec).Return(hookError)
+							if hookError != nil && hook.Exec.OnError == velerov1api.HookErrorModeFail {
+								break hookLoop
+							}
+						}
+						for _, hook := range resourceHook.Post {
+							hookError := pod.hookErrorsByContainer[hook.Exec.Container]
+							podCommandExecutor.On("ExecutePodCommand", mock.Anything, pod.item.UnstructuredContent(), "ns", "name", resourceHook.Name, hook.Exec).Return(hookError)
+							if hookError != nil && hook.Exec.OnError == velerov1api.HookErrorModeFail {
+								break hookLoop
+							}
+						}
+					}
+				}
+				h.HandleHooks(velerotest.NewLogger(), groupResource, pod.item, pod.hooks, test.phase, hookTracker)
+
+			}
+			actualAtemptted, actualFailed := hookTracker.Stat()
+			assert.Equal(t, test.expectedHookAttempted, actualAtemptted)
+			assert.Equal(t, test.expectedHookFailed, actualFailed)
+		})
+	}
+
+}
+
+func TestRestoreHookTrackerAdd(t *testing.T) {
+	testCases := []struct {
+		name                 string
+		resourceRestoreHooks []ResourceRestoreHook
+		pod                  *corev1api.Pod
+		hookTracker          *HookTracker
+		expectedCnt          int
+	}{
+		{
+			name:                 "neither spec hooks nor annotations hooks are set",
+			resourceRestoreHooks: nil,
+			pod:                  builder.ForPod("default", "my-pod").Result(),
+			hookTracker:          NewHookTracker(),
+			expectedCnt:          0,
+		},
+		{
+			name:                 "a hook specified in pod annotation",
+			resourceRestoreHooks: nil,
+			pod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookContainerAnnotationKey, "container1",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+					podRestoreHookWaitForReadyAnnotationKey, "true",
+				)).
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}).
+				Result(),
+			hookTracker: NewHookTracker(),
+			expectedCnt: 1,
+		},
+		{
+			name: "two hooks specified in restore spec",
+			resourceRestoreHooks: []ResourceRestoreHook{
+				{
+					Name:     "hook1",
+					Selector: ResourceHookSelector{},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container1",
+								Command:     []string{"/usr/bin/foo"},
+								OnError:     velerov1api.HookErrorModeContinue,
+								ExecTimeout: metav1.Duration{Duration: time.Second},
+								WaitTimeout: metav1.Duration{Duration: time.Minute},
+							},
+						},
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container2",
+								Command:     []string{"/usr/bin/foo"},
+								OnError:     velerov1api.HookErrorModeContinue,
+								ExecTimeout: metav1.Duration{Duration: time.Second},
+								WaitTimeout: metav1.Duration{Duration: time.Minute},
+							},
+						},
+					},
+				},
+			},
+			pod: builder.ForPod("default", "my-pod").
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}, &corev1api.Container{
+					Name: "container2",
+				}).
+				Result(),
+			hookTracker: NewHookTracker(),
+			expectedCnt: 2,
+		},
+		{
+			name: "both spec hooks and annotations hooks are set",
+			resourceRestoreHooks: []ResourceRestoreHook{
+				{
+					Name:     "hook1",
+					Selector: ResourceHookSelector{},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container1",
+								Command:     []string{"/usr/bin/foo2"},
+								OnError:     velerov1api.HookErrorModeContinue,
+								ExecTimeout: metav1.Duration{Duration: time.Second},
+								WaitTimeout: metav1.Duration{Duration: time.Minute},
+							},
+						},
+					},
+				},
+			},
+			pod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookContainerAnnotationKey, "container1",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+					podRestoreHookWaitForReadyAnnotationKey, "true",
+				)).
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}).
+				Result(),
+			hookTracker: NewHookTracker(),
+			expectedCnt: 1,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, _ = GroupRestoreExecHooks(tc.resourceRestoreHooks, tc.pod, velerotest.NewLogger(), tc.hookTracker)
+			tracker := tc.hookTracker.GetTracker()
+			assert.Equal(t, tc.expectedCnt, len(tracker))
+		})
+	}
+}

internal/hook/wait_exec_hook_handler.go

@@ -39,6 +39,7 @@ type WaitExecHookHandler interface {
 		log logrus.FieldLogger,
 		pod *v1.Pod,
 		byContainer map[string][]PodExecRestoreHook,
+		hookTrack *HookTracker,
 	) []error
 }
 
@@ -73,6 +74,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 	log logrus.FieldLogger,
 	pod *v1.Pod,
 	byContainer map[string][]PodExecRestoreHook,
+	hookTracker *HookTracker,
 ) []error {
 	if pod == nil {
 		return nil
@@ -164,6 +166,8 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 					err := fmt.Errorf("hook %s in container %s expired before executing", hook.HookName, hook.Hook.Container)
 					hookLog.Error(err)
 					errors = append(errors, err)
+					hookTracker.Record(newPod.Namespace, newPod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, hookPhase(""), true)
+
 					if hook.Hook.OnError == velerov1api.HookErrorModeFail {
 						cancel()
 						return
@@ -175,10 +179,13 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 					OnError:   hook.Hook.OnError,
 					Timeout:   hook.Hook.ExecTimeout,
 				}
+				hookTracker.Record(newPod.Namespace, newPod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, hookPhase(""), false)
 				if err := e.PodCommandExecutor.ExecutePodCommand(hookLog, podMap, pod.Namespace, pod.Name, hook.HookName, eh); err != nil {
 					hookLog.WithError(err).Error("Error executing hook")
 					err = fmt.Errorf("hook %s in container %s failed to execute, err: %v", hook.HookName, hook.Hook.Container, err)
 					errors = append(errors, err)
+					hookTracker.Record(newPod.Namespace, newPod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, hookPhase(""), true)
+
 					if hook.Hook.OnError == velerov1api.HookErrorModeFail {
 						cancel()
 						return
@@ -226,6 +233,7 @@ func (e *DefaultWaitExecHookHandler) HandleHooks(
 					"hookPhase":  "post",
 				},
 			)
+			hookTracker.Record(pod.Namespace, pod.Name, hook.Hook.Container, hook.HookSource, hook.HookName, hookPhase(""), true)
 			hookLog.Error(err)
 			errors = append(errors, err)
 		}

internal/hook/wait_exec_hook_handler_test.go

@@ -98,7 +98,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "<from-annotation>",
-						HookSource: "annotation",
+						HookSource: HookSourceAnnotation,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:   "container1",
 							Command:     []string{"/usr/bin/foo"},
@@ -167,7 +167,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "<from-annotation>",
-						HookSource: "annotation",
+						HookSource: HookSourceAnnotation,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:   "container1",
 							Command:     []string{"/usr/bin/foo"},
@@ -236,7 +236,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "<from-annotation>",
-						HookSource: "annotation",
+						HookSource: HookSourceAnnotation,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:   "container1",
 							Command:     []string{"/usr/bin/foo"},
@@ -305,7 +305,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "<from-annotation>",
-						HookSource: "annotation",
+						HookSource: HookSourceAnnotation,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:   "container1",
 							Command:     []string{"/usr/bin/foo"},
@@ -391,7 +391,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "my-hook-1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container: "container1",
 							Command:   []string{"/usr/bin/foo"},
@@ -440,7 +440,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "my-hook-1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:   "container1",
 							Command:     []string{"/usr/bin/foo"},
@@ -471,7 +471,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "my-hook-1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container:   "container1",
 							Command:     []string{"/usr/bin/foo"},
@@ -502,7 +502,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "my-hook-1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container: "container1",
 							Command:   []string{"/usr/bin/foo"},
@@ -533,7 +533,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "my-hook-1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container: "container1",
 							Command:   []string{"/usr/bin/foo"},
@@ -574,7 +574,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container1": {
 					{
 						HookName:   "my-hook-1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container: "container1",
 							Command:   []string{"/usr/bin/foo"},
@@ -584,7 +584,7 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				"container2": {
 					{
 						HookName:   "my-hook-1",
-						HookSource: "backupSpec",
+						HookSource: HookSourceSpec,
 						Hook: velerov1api.ExecRestoreHook{
 							Container: "container2",
 							Command:   []string{"/usr/bin/bar"},
@@ -744,7 +744,8 @@ func TestWaitExecHandleHooks(t *testing.T) {
 				defer ctxCancel()
 			}
 
-			errs := h.HandleHooks(ctx, velerotest.NewLogger(), test.initialPod, test.byContainer)
+			hookTracker := NewHookTracker()
+			errs := h.HandleHooks(ctx, velerotest.NewLogger(), test.initialPod, test.byContainer, hookTracker)
 
 			// for i, ee := range test.expectedErrors {
 			require.Len(t, errs, len(test.expectedErrors))
@@ -997,3 +998,253 @@ func TestMaxHookWait(t *testing.T) {
 		})
 	}
 }
+
+func TestRestoreHookTrackerUpdate(t *testing.T) {
+	type change struct {
+		// delta to wait since last change applied or pod added
+		wait    time.Duration
+		updated *v1.Pod
+	}
+	type expectedExecution struct {
+		hook  *velerov1api.ExecHook
+		name  string
+		error error
+		pod   *v1.Pod
+	}
+
+	hookTracker1 := NewHookTracker()
+	hookTracker1.Add("default", "my-pod", "container1", HookSourceAnnotation, "<from-annotation>", hookPhase(""))
+
+	hookTracker2 := NewHookTracker()
+	hookTracker2.Add("default", "my-pod", "container1", HookSourceSpec, "my-hook-1", hookPhase(""))
+
+	hookTracker3 := NewHookTracker()
+	hookTracker3.Add("default", "my-pod", "container1", HookSourceSpec, "my-hook-1", hookPhase(""))
+	hookTracker3.Add("default", "my-pod", "container2", HookSourceSpec, "my-hook-2", hookPhase(""))
+
+	tests1 := []struct {
+		name               string
+		initialPod         *v1.Pod
+		groupResource      string
+		byContainer        map[string][]PodExecRestoreHook
+		expectedExecutions []expectedExecution
+		hookTracker        *HookTracker
+		expectedFailed     int
+	}{
+		{
+			name: "a hook executes successfully",
+			initialPod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookContainerAnnotationKey, "container1",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+				)).
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Running: &v1.ContainerStateRunning{},
+					},
+				}).
+				Result(),
+			groupResource: "pods",
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "<from-annotation>",
+						HookSource: HookSourceAnnotation,
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{Duration: time.Second},
+							WaitTimeout: metav1.Duration{Duration: time.Minute},
+						},
+					},
+				},
+			},
+			expectedExecutions: []expectedExecution{
+				{
+					name: "<from-annotation>",
+					hook: &velerov1api.ExecHook{
+						Container: "container1",
+						Command:   []string{"/usr/bin/foo"},
+						OnError:   velerov1api.HookErrorModeContinue,
+						Timeout:   metav1.Duration{Duration: time.Second},
+					},
+					error: nil,
+					pod: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("1")).
+						ObjectMeta(builder.WithAnnotations(
+							podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+							podRestoreHookContainerAnnotationKey, "container1",
+							podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+							podRestoreHookTimeoutAnnotationKey, "1s",
+							podRestoreHookWaitTimeoutAnnotationKey, "1m",
+						)).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						Result(),
+				},
+			},
+			hookTracker:    hookTracker1,
+			expectedFailed: 0,
+		},
+		{
+			name:          "a hook with OnError mode Fail failed to execute",
+			groupResource: "pods",
+			initialPod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: HookSourceSpec,
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeFail,
+							WaitTimeout: metav1.Duration{Duration: time.Millisecond},
+						},
+					},
+				},
+			},
+			hookTracker:    hookTracker2,
+			expectedFailed: 1,
+		},
+		{
+			name:          "a hook with OnError mode Continue failed to execute",
+			groupResource: "pods",
+			initialPod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: HookSourceSpec,
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							WaitTimeout: metav1.Duration{Duration: time.Millisecond},
+						},
+					},
+				},
+			},
+			hookTracker:    hookTracker2,
+			expectedFailed: 1,
+		},
+		{
+			name:          "two hooks with OnError mode Continue failed to execute",
+			groupResource: "pods",
+			initialPod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				Containers(&v1.Container{
+					Name: "container2",
+				}).
+				// initially both are waiting
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container2",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: HookSourceSpec,
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							WaitTimeout: metav1.Duration{Duration: time.Millisecond},
+						},
+					},
+				},
+				"container2": {
+					{
+						HookName:   "my-hook-2",
+						HookSource: HookSourceSpec,
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container2",
+							Command:     []string{"/usr/bin/bar"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							WaitTimeout: metav1.Duration{Duration: time.Millisecond},
+						},
+					},
+				},
+			},
+			hookTracker:    hookTracker3,
+			expectedFailed: 2,
+		},
+	}
+
+	for _, test := range tests1 {
+		t.Run(test.name, func(t *testing.T) {
+
+			source := fcache.NewFakeControllerSource()
+			go func() {
+				// This is the state of the pod that will be seen by the AddFunc handler.
+				source.Add(test.initialPod)
+			}()
+
+			podCommandExecutor := &velerotest.MockPodCommandExecutor{}
+			defer podCommandExecutor.AssertExpectations(t)
+
+			h := &DefaultWaitExecHookHandler{
+				PodCommandExecutor: podCommandExecutor,
+				ListWatchFactory:   &fakeListWatchFactory{source},
+			}
+
+			for _, e := range test.expectedExecutions {
+				obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(e.pod)
+				assert.Nil(t, err)
+				podCommandExecutor.On("ExecutePodCommand", mock.Anything, obj, e.pod.Namespace, e.pod.Name, e.name, e.hook).Return(e.error)
+			}
+
+			ctx := context.Background()
+			_ = h.HandleHooks(ctx, velerotest.NewLogger(), test.initialPod, test.byContainer, test.hookTracker)
+			_, actualFailed := test.hookTracker.Stat()
+			assert.Equal(t, test.expectedFailed, actualFailed)
+		})
+	}
+}

pkg/apis/velero/v1/backup_types.go

@@ -176,13 +176,13 @@ type BackupSpec struct {
 	// +optional
 	DataMover string `json:"datamover,omitempty"`
 
-	// BackupConfig defines the configuration for the backup.
+	// UploaderConfig specifies the configuration for the uploader.
 	// +optional
-	BackupConfig *BackupConfig `json:"backupConfig,omitempty"`
+	UploaderConfigForBackup *UploaderConfigForBackup `json:"uploaderConfig,omitempty"`
 }
 
-// BackupConfig defines the configuration for the backup.
-type BackupConfig struct {
+// UploaderConfigForBackup defines the configuration for the backup.
+type UploaderConfigForBackup struct {
 	// ParallelFilesUpload is the number of files parallel uploads to perform when using the uploader.
 	// +optional
 	ParallelFilesUpload int `json:"parallelFilesUpload,omitempty"`
@@ -445,6 +445,11 @@ type BackupStatus struct {
 	// BackupItemAction operations for this backup which ended with an error.
 	// +optional
 	BackupItemOperationsFailed int `json:"backupItemOperationsFailed,omitempty"`
+
+	// HookStatus contains information about the status of the hooks.
+	// +optional
+	// +nullable
+	HookStatus *HookStatus `json:"hookStatus,omitempty"`
 }
 
 // BackupProgress stores information about the progress of a Backup's execution.
@@ -462,6 +467,19 @@ type BackupProgress struct {
 	ItemsBackedUp int `json:"itemsBackedUp,omitempty"`
 }
 
+// HookStatus stores information about the status of the hooks.
+type HookStatus struct {
+	// HooksAttempted is the total number of attempted hooks
+	// Specifically, HooksAttempted represents the number of hooks that failed to execute
+	// and the number of hooks that executed successfully.
+	// +optional
+	HooksAttempted int `json:"hooksAttempted,omitempty"`
+
+	// HooksFailed is the total number of hooks which ended with an error
+	// +optional
+	HooksFailed int `json:"hooksFailed,omitempty"`
+}
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:object:root=true

pkg/apis/velero/v1/pod_volume_backup_types.go

@@ -55,7 +55,7 @@ type PodVolumeBackupSpec struct {
 	// UploaderSettings are a map of key-value pairs that should be applied to the
 	// uploader configuration.
 	// +optional
-	UploaderSettings map[string]string `json:"uploaderSettings,omitempty"`
+	UploaderSettings *map[string]string `json:"uploaderSettings,omitempty"`
 }
 
 // PodVolumeBackupPhase represents the lifecycle phase of a PodVolumeBackup.

pkg/apis/velero/v1/pod_volume_restore_type.go

@@ -52,7 +52,7 @@ type PodVolumeRestoreSpec struct {
 	// UploaderSettings are a map of key-value pairs that should be applied to the
 	// uploader configuration.
 	// +optional
-	UploaderSettings map[string]string `json:"uploaderSettings,omitempty"`
+	UploaderSettings *map[string]string `json:"uploaderSettings,omitempty"`
 }
 
 // PodVolumeRestorePhase represents the lifecycle phase of a PodVolumeRestore.

pkg/apis/velero/v1/restore_types.go

@@ -124,13 +124,13 @@ type RestoreSpec struct {
 	// +nullable
 	ResourceModifier *v1.TypedLocalObjectReference `json:"resourceModifier,omitempty"`
 
-	// RestoreConfig specifies the configuration for the restore.
+	// UploaderConfig specifies the configuration for the restore.
 	// +optional
-	RestoreConfig *RestoreConfig `json:"restoreConfig,omitempty"`
+	UploaderConfigForRestore *UploaderConfigForRestore `json:"uploaderConfig,omitempty"`
 }
 
-// RestoreConfig defines the configuration for the restore.
-type RestoreConfig struct {
+// UploaderConfigForRestore defines the configuration for the restore.
+type UploaderConfigForRestore struct {
 	// WriteSparseFiles is a flag to indicate whether write files sparsely or not.
 	// +optional
 	WriteSparseFiles bool `json:"writeSparseFiles,omitempty"`
@@ -356,6 +356,11 @@ type RestoreStatus struct {
 	// RestoreItemAction operations for this restore which ended with an error.
 	// +optional
 	RestoreItemOperationsFailed int `json:"restoreItemOperationsFailed,omitempty"`
+
+	// HookStatus contains information about the status of the hooks.
+	// +optional
+	// +nullable
+	HookStatus *HookStatus `json:"hookStatus,omitempty"`
 }
 
 // RestoreProgress stores information about the restore's execution progress

pkg/apis/velero/v1/zz_generated.deepcopy.go

@@ -38,21 +38,6 @@ func (in *Backup) DeepCopyObject() runtime.Object {
 	return nil
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *BackupConfig) DeepCopyInto(out *BackupConfig) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupConfig.
-func (in *BackupConfig) DeepCopy() *BackupConfig {
-	if in == nil {
-		return nil
-	}
-	out := new(BackupConfig)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *BackupHooks) DeepCopyInto(out *BackupHooks) {
 	*out = *in
@@ -396,9 +381,9 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) {
 		*out = new(bool)
 		**out = **in
 	}
-	if in.BackupConfig != nil {
-		in, out := &in.BackupConfig, &out.BackupConfig
-		*out = new(BackupConfig)
+	if in.UploaderConfigForBackup != nil {
+		in, out := &in.UploaderConfigForBackup, &out.UploaderConfigForBackup
+		*out = new(UploaderConfigForBackup)
 		**out = **in
 	}
 }
@@ -438,6 +423,11 @@ func (in *BackupStatus) DeepCopyInto(out *BackupStatus) {
 		*out = new(BackupProgress)
 		**out = **in
 	}
+	if in.HookStatus != nil {
+		in, out := &in.HookStatus, &out.HookStatus
+		*out = new(HookStatus)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStatus.
@@ -821,6 +811,21 @@ func (in *ExecRestoreHook) DeepCopy() *ExecRestoreHook {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HookStatus) DeepCopyInto(out *HookStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HookStatus.
+func (in *HookStatus) DeepCopy() *HookStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(HookStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *InitRestoreHook) DeepCopyInto(out *InitRestoreHook) {
 	*out = *in
@@ -973,9 +978,13 @@ func (in *PodVolumeBackupSpec) DeepCopyInto(out *PodVolumeBackupSpec) {
 	}
 	if in.UploaderSettings != nil {
 		in, out := &in.UploaderSettings, &out.UploaderSettings
-		*out = make(map[string]string, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
+		*out = new(map[string]string)
+		if **in != nil {
+			in, out := *in, *out
+			*out = make(map[string]string, len(*in))
+			for key, val := range *in {
+				(*out)[key] = val
+			}
 		}
 	}
 }
@@ -1079,9 +1088,13 @@ func (in *PodVolumeRestoreSpec) DeepCopyInto(out *PodVolumeRestoreSpec) {
 	out.Pod = in.Pod
 	if in.UploaderSettings != nil {
 		in, out := &in.UploaderSettings, &out.UploaderSettings
-		*out = make(map[string]string, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
+		*out = new(map[string]string)
+		if **in != nil {
+			in, out := *in, *out
+			*out = make(map[string]string, len(*in))
+			for key, val := range *in {
+				(*out)[key] = val
+			}
 		}
 	}
 }
@@ -1147,21 +1160,6 @@ func (in *Restore) DeepCopyObject() runtime.Object {
 	return nil
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *RestoreConfig) DeepCopyInto(out *RestoreConfig) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreConfig.
-func (in *RestoreConfig) DeepCopy() *RestoreConfig {
-	if in == nil {
-		return nil
-	}
-	out := new(RestoreConfig)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RestoreHooks) DeepCopyInto(out *RestoreHooks) {
 	*out = *in
@@ -1376,9 +1374,9 @@ func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) {
 		*out = new(corev1.TypedLocalObjectReference)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.RestoreConfig != nil {
-		in, out := &in.RestoreConfig, &out.RestoreConfig
-		*out = new(RestoreConfig)
+	if in.UploaderConfigForRestore != nil {
+		in, out := &in.UploaderConfigForRestore, &out.UploaderConfigForRestore
+		*out = new(UploaderConfigForRestore)
 		**out = **in
 	}
 }
@@ -1414,6 +1412,11 @@ func (in *RestoreStatus) DeepCopyInto(out *RestoreStatus) {
 		*out = new(RestoreProgress)
 		**out = **in
 	}
+	if in.HookStatus != nil {
+		in, out := &in.HookStatus, &out.HookStatus
+		*out = new(HookStatus)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreStatus.
@@ -1673,6 +1676,36 @@ func (in *StorageType) DeepCopy() *StorageType {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UploaderConfigForBackup) DeepCopyInto(out *UploaderConfigForBackup) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UploaderConfigForBackup.
+func (in *UploaderConfigForBackup) DeepCopy() *UploaderConfigForBackup {
+	if in == nil {
+		return nil
+	}
+	out := new(UploaderConfigForBackup)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UploaderConfigForRestore) DeepCopyInto(out *UploaderConfigForRestore) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UploaderConfigForRestore.
+func (in *UploaderConfigForRestore) DeepCopy() *UploaderConfigForRestore {
+	if in == nil {
+		return nil
+	}
+	out := new(UploaderConfigForRestore)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *VolumeSnapshotLocation) DeepCopyInto(out *VolumeSnapshotLocation) {
 	*out = *in

pkg/backup/backup.go

@@ -302,6 +302,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger,
 		itemHookHandler: &hook.DefaultItemHookHandler{
 			PodCommandExecutor: kb.podCommandExecutor,
 		},
+		hookTracker: hook.NewHookTracker(),
 	}
 
 	// helper struct to send current progress between the main
@@ -427,8 +428,15 @@ func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger,
 	updated.Status.Progress.TotalItems = len(backupRequest.BackedUpItems)
 	updated.Status.Progress.ItemsBackedUp = len(backupRequest.BackedUpItems)
 
+	// update the hooks execution status
+	if updated.Status.HookStatus == nil {
+		updated.Status.HookStatus = &velerov1api.HookStatus{}
+	}
+	updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed = itemBackupper.hookTracker.Stat()
+	log.Infof("hookTracker: %+v, hookAttempted: %d, hookFailed: %d", itemBackupper.hookTracker.GetTracker(), updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed)
+
 	if err := kube.PatchResource(backupRequest.Backup, updated, kb.kbClient); err != nil {
-		log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress")
+		log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress and hook status")
 	}
 	skippedPVSummary, _ := json.Marshal(backupRequest.SkippedPVTracker.Summary())
 	log.Infof("Summary for skipped PVs: %s", skippedPVSummary)
@@ -598,6 +606,7 @@ func (kb *kubernetesBackupper) FinalizeBackup(log logrus.FieldLogger,
 		discoveryHelper:          kb.discoveryHelper,
 		itemHookHandler:          &hook.NoOpItemHookHandler{},
 		podVolumeSnapshotTracker: newPVCSnapshotTracker(),
+		hookTracker:              hook.NewHookTracker(),
 	}
 	updateFiles := make(map[string]FileForArchive)
 	backedUpGroupResources := map[schema.GroupResource]bool{}

pkg/backup/item_backupper.go

@@ -78,6 +78,7 @@ type itemBackupper struct {
 
 	itemHookHandler                    hook.ItemHookHandler
 	snapshotLocationVolumeSnapshotters map[string]vsv1.VolumeSnapshotter
+	hookTracker                        *hook.HookTracker
 }
 
 type FileForArchive struct {
@@ -184,7 +185,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti
 	)
 
 	log.Debug("Executing pre hooks")
-	if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePre); err != nil {
+	if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePre, ib.hookTracker); err != nil {
 		return false, itemFiles, err
 	}
 	if optedOut, podName := ib.podVolumeSnapshotTracker.OptedoutByPod(namespace, name); optedOut {
@@ -234,7 +235,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti
 
 		// if there was an error running actions, execute post hooks and return
 		log.Debug("Executing post hooks")
-		if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePost); err != nil {
+		if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePost, ib.hookTracker); err != nil {
 			backupErrs = append(backupErrs, err)
 		}
 		return false, itemFiles, kubeerrs.NewAggregate(backupErrs)
@@ -293,7 +294,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti
 	}
 
 	log.Debug("Executing post hooks")
-	if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePost); err != nil {
+	if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePost, ib.hookTracker); err != nil {
 		backupErrs = append(backupErrs, err)
 	}
 

pkg/builder/backup_builder.go

@@ -302,10 +302,10 @@ func (b *BackupBuilder) DataMover(name string) *BackupBuilder {
 
 // ParallelFilesUpload sets the Backup's uploader parallel uploads
 func (b *BackupBuilder) ParallelFilesUpload(parallel int) *BackupBuilder {
-	if b.object.Spec.BackupConfig == nil {
-		b.object.Spec.BackupConfig = &velerov1api.BackupConfig{}
+	if b.object.Spec.UploaderConfigForBackup == nil {
+		b.object.Spec.UploaderConfigForBackup = &velerov1api.UploaderConfigForBackup{}
 	}
-	b.object.Spec.BackupConfig.ParallelFilesUpload = parallel
+	b.object.Spec.UploaderConfigForBackup.ParallelFilesUpload = parallel
 	return b
 }
 

pkg/builder/restore_builder.go

@@ -174,6 +174,6 @@ func (b *RestoreBuilder) ItemOperationTimeout(timeout time.Duration) *RestoreBui
 
 // WriteSparseFiles sets the Restore's uploader write sparse files
 func (b *RestoreBuilder) WriteSparseFiles(val bool) *RestoreBuilder {
-	b.object.Spec.RestoreConfig.WriteSparseFiles = val
+	b.object.Spec.UploaderConfigForRestore.WriteSparseFiles = val
 	return b
 }

pkg/cmd/cli/install/install.go

@@ -73,6 +73,7 @@ type Options struct {
 	UseVolumeSnapshots              bool
 	DefaultRepoMaintenanceFrequency time.Duration
 	GarbageCollectionFrequency      time.Duration
+	PodVolumeOperationTimeout       time.Duration
 	Plugins                         flag.StringArray
 	NoDefaultBackupLocation         bool
 	CRDsOnly                        bool
@@ -116,6 +117,7 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) {
 	flags.BoolVar(&o.Wait, "wait", o.Wait, "Wait for Velero deployment to be ready. Optional.")
 	flags.DurationVar(&o.DefaultRepoMaintenanceFrequency, "default-repo-maintain-frequency", o.DefaultRepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default. Optional.")
 	flags.DurationVar(&o.GarbageCollectionFrequency, "garbage-collection-frequency", o.GarbageCollectionFrequency, "How often the garbage collection runs for expired backups.(default 1h)")
+	flags.DurationVar(&o.PodVolumeOperationTimeout, "pod-volume-operation-timeout", o.PodVolumeOperationTimeout, "How long to wait for pod volume operations to complete before timing out(default 4h). Optional.")
 	flags.Var(&o.Plugins, "plugins", "Plugin container images to install into the Velero Deployment")
 	flags.BoolVar(&o.CRDsOnly, "crds-only", o.CRDsOnly, "Only generate CustomResourceDefinition resources. Useful for updating CRDs for an existing Velero install.")
 	flags.StringVar(&o.CACertFile, "cacert", o.CACertFile, "File containing a certificate bundle to use when verifying TLS connections to the object store. Optional.")
@@ -209,6 +211,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) {
 		VSLConfig:                       o.VolumeSnapshotConfig.Data(),
 		DefaultRepoMaintenanceFrequency: o.DefaultRepoMaintenanceFrequency,
 		GarbageCollectionFrequency:      o.GarbageCollectionFrequency,
+		PodVolumeOperationTimeout:       o.PodVolumeOperationTimeout,
 		Plugins:                         o.Plugins,
 		NoDefaultBackupLocation:         o.NoDefaultBackupLocation,
 		CACertData:                      caCertData,
@@ -426,5 +429,9 @@ func (o *Options) Validate(c *cobra.Command, args []string, f client.Factory) er
 		return errors.New("--garbage-collection-frequency must be non-negative")
 	}
 
+	if o.PodVolumeOperationTimeout < 0 {
+		return errors.New("--pod-volume-operation-timeout must be non-negative")
+	}
+
 	return nil
 }

pkg/cmd/cli/nodeagent/server.go

@@ -194,7 +194,6 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi
 		Scheme:   scheme,
 		NewCache: cache.BuilderWithOptions(cacheOption),
 	})
-
 	if err != nil {
 		cancelFunc()
 		return nil, err
@@ -286,13 +285,13 @@ func (s *nodeAgentServer) run() {
 	}
 
 	dataUploadReconciler := controller.NewDataUploadReconciler(s.mgr.GetClient(), s.kubeClient, s.csiSnapshotClient.SnapshotV1(), s.dataPathMgr, repoEnsurer, clock.RealClock{}, credentialGetter, s.nodeName, s.fileSystem, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
-	s.markDataUploadsCancel(dataUploadReconciler)
+	s.attemptDataUploadResume(dataUploadReconciler)
 	if err = dataUploadReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the data upload controller")
 	}
 
 	dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.kubeClient, s.dataPathMgr, repoEnsurer, credentialGetter, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
-	s.markDataDownloadsCancel(dataDownloadReconciler)
+	s.attemptDataDownloadResume(dataDownloadReconciler)
 	if err = dataDownloadReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the data download controller")
 	}
@@ -366,71 +365,34 @@ func (s *nodeAgentServer) markInProgressCRsFailed() {
 	s.markInProgressPVRsFailed(client)
 }
 
-func (s *nodeAgentServer) markDataUploadsCancel(r *controller.DataUploadReconciler) {
+func (s *nodeAgentServer) attemptDataUploadResume(r *controller.DataUploadReconciler) {
 	// the function is called before starting the controller manager, the embedded client isn't ready to use, so create a new one here
 	client, err := ctrlclient.New(s.mgr.GetConfig(), ctrlclient.Options{Scheme: s.mgr.GetScheme()})
 	if err != nil {
 		s.logger.WithError(errors.WithStack(err)).Error("failed to create client")
 		return
 	}
-	if dataUploads, err := r.FindDataUploads(s.ctx, client, s.namespace); err != nil {
-		s.logger.WithError(errors.WithStack(err)).Error("failed to find data uploads")
-	} else {
-		for i := range dataUploads {
-			du := dataUploads[i]
-			if du.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted ||
-				du.Status.Phase == velerov2alpha1api.DataUploadPhasePrepared ||
-				du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress {
-				err = controller.UpdateDataUploadWithRetry(s.ctx, client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, s.logger.WithField("dataupload", du.Name),
-					func(dataUpload *velerov2alpha1api.DataUpload) {
-						dataUpload.Spec.Cancel = true
-						dataUpload.Status.Message = fmt.Sprintf("found a dataupload with status %q during the node-agent starting, mark it as cancel", du.Status.Phase)
-					})
-
-				if err != nil {
-					s.logger.WithError(errors.WithStack(err)).Errorf("failed to mark dataupload %q cancel", du.GetName())
-					continue
-				}
-				s.logger.WithField("dataupload", du.GetName()).Warn(du.Status.Message)
-			}
-		}
+	if err := r.AttemptDataUploadResume(s.ctx, client, s.logger.WithField("node", s.nodeName), s.namespace); err != nil {
+		s.logger.WithError(errors.WithStack(err)).Error("failed to attempt data upload resume")
 	}
 }
 
-func (s *nodeAgentServer) markDataDownloadsCancel(r *controller.DataDownloadReconciler) {
+func (s *nodeAgentServer) attemptDataDownloadResume(r *controller.DataDownloadReconciler) {
 	// the function is called before starting the controller manager, the embedded client isn't ready to use, so create a new one here
 	client, err := ctrlclient.New(s.mgr.GetConfig(), ctrlclient.Options{Scheme: s.mgr.GetScheme()})
 	if err != nil {
 		s.logger.WithError(errors.WithStack(err)).Error("failed to create client")
 		return
 	}
-	if dataDownloads, err := r.FindDataDownloads(s.ctx, client, s.namespace); err != nil {
-		s.logger.WithError(errors.WithStack(err)).Error("failed to find data downloads")
-	} else {
-		for i := range dataDownloads {
-			dd := dataDownloads[i]
-			if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted ||
-				dd.Status.Phase == velerov2alpha1api.DataDownloadPhasePrepared ||
-				dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress {
-				err = controller.UpdateDataDownloadWithRetry(s.ctx, client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, s.logger.WithField("datadownload", dd.Name),
-					func(dataDownload *velerov2alpha1api.DataDownload) {
-						dataDownload.Spec.Cancel = true
-						dataDownload.Status.Message = fmt.Sprintf("found a datadownload with status %q during the node-agent starting, mark it as cancel", dd.Status.Phase)
-					})
 
-				if err != nil {
-					s.logger.WithError(errors.WithStack(err)).Errorf("failed to mark datadownload %q cancel", dd.GetName())
-					continue
-				}
-				s.logger.WithField("datadownload", dd.GetName()).Warn(dd.Status.Message)
-			}
-		}
+	if err := r.AttemptDataDownloadResume(s.ctx, client, s.logger.WithField("node", s.nodeName), s.namespace); err != nil {
+		s.logger.WithError(errors.WithStack(err)).Error("failed to attempt data download resume")
 	}
 }
 
 func (s *nodeAgentServer) markInProgressPVBsFailed(client ctrlclient.Client) {
 	pvbs := &velerov1api.PodVolumeBackupList{}
-	if err := client.List(s.ctx, pvbs, &ctrlclient.MatchingFields{"metadata.namespace": s.namespace}); err != nil {
+	if err := client.List(s.ctx, pvbs, &ctrlclient.ListOptions{Namespace: s.namespace}); err != nil {
 		s.logger.WithError(errors.WithStack(err)).Error("failed to list podvolumebackups")
 		return
 	}
@@ -456,7 +418,7 @@ func (s *nodeAgentServer) markInProgressPVBsFailed(client ctrlclient.Client) {
 
 func (s *nodeAgentServer) markInProgressPVRsFailed(client ctrlclient.Client) {
 	pvrs := &velerov1api.PodVolumeRestoreList{}
-	if err := client.List(s.ctx, pvrs, &ctrlclient.MatchingFields{"metadata.namespace": s.namespace}); err != nil {
+	if err := client.List(s.ctx, pvrs, &ctrlclient.ListOptions{Namespace: s.namespace}); err != nil {
 		s.logger.WithError(errors.WithStack(err)).Error("failed to list podvolumerestores")
 		return
 	}

pkg/cmd/cli/restore/create.go

@@ -149,7 +149,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
 
 	flags.StringVar(&o.ResourceModifierConfigMap, "resource-modifier-configmap", "", "Reference to the resource modifier configmap that restore will use")
 
-	f = flags.VarPF(&o.WriteSparseFiles, "write-sparse-files", "", "Whether to write sparse files when restoring volumes using restic or kopia")
+	f = flags.VarPF(&o.WriteSparseFiles, "write-sparse-files", "", "Whether to write sparse files when restoring volumes")
 	f.NoOptDefVal = cmd.FALSE
 }
 
@@ -323,7 +323,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 			ItemOperationTimeout: metav1.Duration{
 				Duration: o.ItemOperationTimeout,
 			},
-			RestoreConfig: &api.RestoreConfig{
+			UploaderConfigForRestore: &api.UploaderConfigForRestore{
 				WriteSparseFiles: boolptr.IsSetToTrue(o.WriteSparseFiles.Value),
 			},
 		},

pkg/cmd/server/server.go

@@ -34,6 +34,7 @@ import (
 	"github.com/spf13/cobra"
 	corev1api "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	kubeerrs "k8s.io/apimachinery/pkg/util/errors"
@@ -990,7 +991,7 @@ func markInProgressCRsFailed(ctx context.Context, cfg *rest.Config, scheme *runt
 
 func markInProgressBackupsFailed(ctx context.Context, client ctrlclient.Client, namespace string, log logrus.FieldLogger) {
 	backups := &velerov1api.BackupList{}
-	if err := client.List(ctx, backups, &ctrlclient.MatchingFields{"metadata.namespace": namespace}); err != nil {
+	if err := client.List(ctx, backups, &ctrlclient.ListOptions{Namespace: namespace}); err != nil {
 		log.WithError(errors.WithStack(err)).Error("failed to list backups")
 		return
 	}
@@ -1015,7 +1016,7 @@ func markInProgressBackupsFailed(ctx context.Context, client ctrlclient.Client,
 
 func markInProgressRestoresFailed(ctx context.Context, client ctrlclient.Client, namespace string, log logrus.FieldLogger) {
 	restores := &velerov1api.RestoreList{}
-	if err := client.List(ctx, restores, &ctrlclient.MatchingFields{"metadata.namespace": namespace}); err != nil {
+	if err := client.List(ctx, restores, &ctrlclient.ListOptions{Namespace: namespace}); err != nil {
 		log.WithError(errors.WithStack(err)).Error("failed to list restores")
 		return
 	}
@@ -1040,7 +1041,12 @@ func markInProgressRestoresFailed(ctx context.Context, client ctrlclient.Client,
 func markDataUploadsCancel(ctx context.Context, client ctrlclient.Client, backup velerov1api.Backup, log logrus.FieldLogger) {
 	dataUploads := &velerov2alpha1api.DataUploadList{}
 
-	if err := client.List(ctx, dataUploads, &ctrlclient.MatchingFields{"metadata.namespace": backup.GetNamespace()}, &ctrlclient.MatchingLabels{velerov1api.BackupUIDLabel: string(backup.GetUID())}); err != nil {
+	if err := client.List(ctx, dataUploads, &ctrlclient.ListOptions{
+		Namespace: backup.GetNamespace(),
+		LabelSelector: labels.Set(map[string]string{
+			velerov1api.BackupUIDLabel: string(backup.GetUID()),
+		}).AsSelector(),
+	}); err != nil {
 		log.WithError(errors.WithStack(err)).Error("failed to list dataUploads")
 		return
 	}
@@ -1070,7 +1076,12 @@ func markDataUploadsCancel(ctx context.Context, client ctrlclient.Client, backup
 func markDataDownloadsCancel(ctx context.Context, client ctrlclient.Client, restore velerov1api.Restore, log logrus.FieldLogger) {
 	dataDownloads := &velerov2alpha1api.DataDownloadList{}
 
-	if err := client.List(ctx, dataDownloads, &ctrlclient.MatchingFields{"metadata.namespace": restore.GetNamespace()}, &ctrlclient.MatchingLabels{velerov1api.RestoreUIDLabel: string(restore.GetUID())}); err != nil {
+	if err := client.List(ctx, dataDownloads, &ctrlclient.ListOptions{
+		Namespace: restore.GetNamespace(),
+		LabelSelector: labels.Set(map[string]string{
+			velerov1api.RestoreUIDLabel: string(restore.GetUID()),
+		}).AsSelector(),
+	}); err != nil {
 		log.WithError(errors.WithStack(err)).Error("failed to list dataDownloads")
 		return
 	}

pkg/cmd/util/output/backup_describer.go

@@ -88,7 +88,7 @@ func DescribeBackup(
 			DescribeResourcePolicies(d, backup.Spec.ResourcePolicy)
 		}
 
-		if backup.Spec.BackupConfig != nil && backup.Spec.BackupConfig.ParallelFilesUpload > 0 {
+		if backup.Spec.UploaderConfigForBackup != nil && backup.Spec.UploaderConfigForBackup.ParallelFilesUpload > 0 {
 			d.Println()
 			DescribeUploaderConfig(d, backup.Spec)
 		}
@@ -138,7 +138,7 @@ func DescribeResourcePolicies(d *Describer, resPolicies *v1.TypedLocalObjectRefe
 // DescribeUploaderConfig describes uploader config in human-readable format
 func DescribeUploaderConfig(d *Describer, spec velerov1api.BackupSpec) {
 	d.Printf("Uploader config:\n")
-	d.Printf("\tParallel files upload:\t%d\n", spec.BackupConfig.ParallelFilesUpload)
+	d.Printf("\tParallel files upload:\t%d\n", spec.UploaderConfigForBackup.ParallelFilesUpload)
 }
 
 // DescribeBackupSpec describes a backup spec in human-readable format.
@@ -392,6 +392,12 @@ func DescribeBackupStatus(ctx context.Context, kbClient kbclient.Client, d *Desc
 	}
 
 	d.Printf("Velero-Native Snapshots: <none included>\n")
+
+	if status.HookStatus != nil {
+		d.Println()
+		d.Printf("HooksAttempted:\t%d\n", status.HookStatus.HooksAttempted)
+		d.Printf("HooksFailed:\t%d\n", status.HookStatus.HooksFailed)
+	}
 }
 
 func describeBackupItemOperations(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string) {

pkg/cmd/util/output/backup_structured_describer.go

@@ -303,6 +303,11 @@ func DescribeBackupStatusInSF(ctx context.Context, kbClient kbclient.Client, d *
 		backupStatusInfo["veleroNativeSnapshotsDetail"] = snapshotDetails
 		return
 	}
+
+	if status.HookStatus != nil {
+		backupStatusInfo["hooksAttempted"] = status.HookStatus.HooksAttempted
+		backupStatusInfo["hooksFailed"] = status.HookStatus.HooksFailed
+	}
 }
 
 func describeBackupResourceListInSF(ctx context.Context, kbClient kbclient.Client, backupStatusInfo map[string]interface{}, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) {

pkg/cmd/util/output/restore_describer.go

@@ -177,14 +177,20 @@ func DescribeRestore(ctx context.Context, kbClient kbclient.Client, restore *vel
 		d.Println()
 		d.Printf("Preserve Service NodePorts:\t%s\n", BoolPointerString(restore.Spec.PreserveNodePorts, "false", "true", "auto"))
 
-		if restore.Spec.RestoreConfig != nil && restore.Spec.RestoreConfig.WriteSparseFiles {
+		if restore.Spec.UploaderConfigForRestore != nil && restore.Spec.UploaderConfigForRestore.WriteSparseFiles {
 			d.Println()
-			d.Printf("Write Sparse Files:\t%T\n", restore.Spec.RestoreConfig.WriteSparseFiles)
+			d.Printf("Write Sparse Files:\t%T\n", restore.Spec.UploaderConfigForRestore.WriteSparseFiles)
 		}
 
 		d.Println()
 		describeRestoreItemOperations(ctx, kbClient, d, restore, details, insecureSkipTLSVerify, caCertFile)
 
+		if restore.Status.HookStatus != nil {
+			d.Println()
+			d.Printf("HooksAttempted: \t%d\n", restore.Status.HookStatus.HooksAttempted)
+			d.Printf("HooksFailed: \t%d\n", restore.Status.HookStatus.HooksFailed)
+		}
+
 		if details {
 			describeRestoreResourceList(ctx, kbClient, d, restore, insecureSkipTLSVerify, caCertFile)
 			d.Println()

pkg/cmd/util/output/schedule_describer.go

@@ -48,7 +48,7 @@ func DescribeSchedule(schedule *v1.Schedule) string {
 			DescribeResourcePolicies(d, schedule.Spec.Template.ResourcePolicy)
 		}
 
-		if schedule.Spec.Template.BackupConfig != nil && schedule.Spec.Template.BackupConfig.ParallelFilesUpload > 0 {
+		if schedule.Spec.Template.UploaderConfigForBackup != nil && schedule.Spec.Template.UploaderConfigForBackup.ParallelFilesUpload > 0 {
 			d.Println()
 			DescribeUploaderConfig(d, schedule.Spec.Template)
 		}

pkg/controller/data_download_controller.go

@@ -140,7 +140,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		// to help clear up resources instead of clear them directly in case of some conflict with Expose action
 		if err := UpdateDataDownloadWithRetry(ctx, r.client, req.NamespacedName, log, func(dataDownload *velerov2alpha1api.DataDownload) {
 			dataDownload.Spec.Cancel = true
-			dataDownload.Status.Message = fmt.Sprintf("found a dataupload %s/%s is being deleted, mark it as cancel", dd.Namespace, dd.Name)
+			dataDownload.Status.Message = fmt.Sprintf("found a datadownload %s/%s is being deleted, mark it as cancel", dd.Namespace, dd.Name)
 		}); err != nil {
 			log.Errorf("failed to set cancel flag with error %s for %s/%s", err.Error(), dd.Namespace, dd.Name)
 			return ctrl.Result{}, err
@@ -192,7 +192,6 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
 				return r.errorOut(ctx, dd, err, "error to expose snapshot", log)
 			}
 		}
-
 		log.Info("Restore is exposed")
 
 		// we need to get CR again for it may canceled by datadownload controller on other
@@ -205,7 +204,6 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
 			}
 			return ctrl.Result{}, errors.Wrap(err, "getting datadownload")
 		}
-
 		// we need to clean up resources as resources created in Expose it may later than cancel action or prepare time
 		// and need to clean up resources again
 		if isDataDownloadInFinalState(dd) {
@@ -267,7 +265,6 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
 				return r.errorOut(ctx, dd, err, "error to create data path", log)
 			}
 		}
-
 		// Update status to InProgress
 		original := dd.DeepCopy()
 		dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseInProgress
@@ -335,7 +332,7 @@ func (r *DataDownloadReconciler) runCancelableDataPath(ctx context.Context, fsRe
 	}
 	log.WithField("path", path.ByPath).Info("fs init")
 
-	if err := fsRestore.StartRestore(dd.Spec.SnapshotID, path, dd.Spec.DataMoverConfig); err != nil {
+	if err := fsRestore.StartRestore(dd.Spec.SnapshotID, path, &dd.Spec.DataMoverConfig); err != nil {
 		return r.errorOut(ctx, dd, err, fmt.Sprintf("error starting data path %s restore", path.ByPath), log)
 	}
 
@@ -576,6 +573,51 @@ func (r *DataDownloadReconciler) FindDataDownloads(ctx context.Context, cli clie
 	return dataDownloads, nil
 }
 
+func (r *DataDownloadReconciler) findAcceptDataDownloadsByNodeLabel(ctx context.Context, cli client.Client, ns string) ([]velerov2alpha1api.DataDownload, error) {
+	dataDownloads := &velerov2alpha1api.DataDownloadList{}
+	if err := cli.List(ctx, dataDownloads, &client.ListOptions{Namespace: ns}); err != nil {
+		r.logger.WithError(errors.WithStack(err)).Error("failed to list datauploads")
+		return nil, errors.Wrapf(err, "failed to list datauploads")
+	}
+
+	var result []velerov2alpha1api.DataDownload
+	for _, dd := range dataDownloads.Items {
+		if dd.Status.Phase != velerov2alpha1api.DataDownloadPhaseAccepted {
+			continue
+		}
+		if dd.Labels[acceptNodeLabelKey] == r.nodeName {
+			result = append(result, dd)
+		}
+	}
+	return result, nil
+}
+
+// CancelAcceptedDataDownload will cancel the accepted data download
+func (r *DataDownloadReconciler) CancelAcceptedDataDownload(ctx context.Context, cli client.Client, ns string) {
+	r.logger.Infof("Canceling accepted data for node %s", r.nodeName)
+	dataDownloads, err := r.findAcceptDataDownloadsByNodeLabel(ctx, cli, ns)
+	if err != nil {
+		r.logger.WithError(err).Error("failed to find data downloads")
+		return
+	}
+
+	for _, dd := range dataDownloads {
+		if dd.Spec.Cancel {
+			continue
+		}
+		err = UpdateDataDownloadWithRetry(ctx, cli, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name},
+			r.logger.WithField("dataupload", dd.Name), func(dataDownload *velerov2alpha1api.DataDownload) {
+				dataDownload.Spec.Cancel = true
+				dataDownload.Status.Message = fmt.Sprintf("found a datadownload with status %q during the node-agent starting, mark it as cancel", dd.Status.Phase)
+			})
+
+		r.logger.Warn(dd.Status.Message)
+		if err != nil {
+			r.logger.WithError(err).Errorf("failed to set cancel flag with error %s", err.Error())
+		}
+	}
+}
+
 func (r *DataDownloadReconciler) prepareDataDownload(ssb *velerov2alpha1api.DataDownload) {
 	ssb.Status.Phase = velerov2alpha1api.DataDownloadPhasePrepared
 	ssb.Status.Node = r.nodeName
@@ -749,3 +791,35 @@ func UpdateDataDownloadWithRetry(ctx context.Context, client client.Client, name
 		return true, nil
 	})
 }
+
+func (r *DataDownloadReconciler) AttemptDataDownloadResume(ctx context.Context, cli client.Client, logger *logrus.Entry, ns string) error {
+	if dataDownloads, err := r.FindDataDownloads(ctx, cli, ns); err != nil {
+		return errors.Wrapf(err, "failed to find data downloads")
+	} else {
+		for i := range dataDownloads {
+			dd := dataDownloads[i]
+			if dd.Status.Phase == velerov2alpha1api.DataDownloadPhasePrepared {
+				// keep doing nothing let controller re-download the data
+				// the Prepared CR could be still handled by datadownload controller after node-agent restart
+				logger.WithField("datadownload", dd.GetName()).Debug("find a datadownload with status prepared")
+			} else if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress {
+				err = UpdateDataDownloadWithRetry(ctx, cli, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, logger.WithField("datadownload", dd.Name),
+					func(dataDownload *velerov2alpha1api.DataDownload) {
+						dataDownload.Spec.Cancel = true
+						dataDownload.Status.Message = fmt.Sprintf("found a datadownload with status %q during the node-agent starting, mark it as cancel", dd.Status.Phase)
+					})
+
+				if err != nil {
+					logger.WithError(errors.WithStack(err)).Errorf("failed to mark datadownload %q into canceled", dd.GetName())
+					continue
+				}
+				logger.WithField("datadownload", dd.GetName()).Debug("mark datadownload into canceled")
+			}
+		}
+	}
+
+	//If the data download is in Accepted status, the expoded PVC may be not created
+	// so we need to mark the data download as canceled for it may not be recoverable
+	r.CancelAcceptedDataDownload(ctx, cli, ns)
+	return nil
+}

pkg/controller/data_download_controller_test.go

@@ -69,7 +69,7 @@ func dataDownloadBuilder() *builder.DataDownloadBuilder {
 }
 
 func initDataDownloadReconciler(objects []runtime.Object, needError ...bool) (*DataDownloadReconciler, error) {
-	var errs []error = make([]error, 5)
+	var errs []error = make([]error, 6)
 	for k, isError := range needError {
 		if k == 0 && isError {
 			errs[0] = fmt.Errorf("Get error")
@@ -81,6 +81,8 @@ func initDataDownloadReconciler(objects []runtime.Object, needError ...bool) (*D
 			errs[3] = fmt.Errorf("Patch error")
 		} else if k == 4 && isError {
 			errs[4] = apierrors.NewConflict(velerov2alpha1api.Resource("datadownload"), dataDownloadName, errors.New("conflict"))
+		} else if k == 5 && isError {
+			errs[5] = fmt.Errorf("List error")
 		}
 	}
 	return initDataDownloadReconcilerWithError(objects, errs...)
@@ -116,6 +118,8 @@ func initDataDownloadReconcilerWithError(objects []runtime.Object, needError ...
 			fakeClient.patchError = needError[3]
 		} else if k == 4 {
 			fakeClient.updateConflict = needError[4]
+		} else if k == 5 {
+			fakeClient.listError = needError[5]
 		}
 	}
 
@@ -939,3 +943,111 @@ func TestFindDataDownloads(t *testing.T) {
 		})
 	}
 }
+
+func TestAttemptDataDownloadResume(t *testing.T) {
+	tests := []struct {
+		name                   string
+		dataUploads            []velerov2alpha1api.DataDownload
+		du                     *velerov2alpha1api.DataDownload
+		pod                    *corev1.Pod
+		needErrs               []bool
+		acceptedDataDownloads  []string
+		prepareddDataDownloads []string
+		cancelledDataDownloads []string
+		expectedError          bool
+	}{
+		// Test case 1: Process Accepted DataDownload
+		{
+			name: "AcceptedDataDownload",
+			pod: builder.ForPod(velerov1api.DefaultNamespace, dataDownloadName).Volumes(&corev1.Volume{Name: dataDownloadName}).NodeName("node-1").Labels(map[string]string{
+				velerov1api.DataDownloadLabel: dataDownloadName,
+			}).Result(),
+			du:                    dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(),
+			acceptedDataDownloads: []string{dataDownloadName},
+			expectedError:         false,
+		},
+		// Test case 2: Cancel an Accepted DataDownload
+		{
+			name: "CancelAcceptedDataDownload",
+			du:   dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(),
+		},
+		// Test case 3: Process Accepted Prepared DataDownload
+		{
+			name: "PreparedDataDownload",
+			pod: builder.ForPod(velerov1api.DefaultNamespace, dataDownloadName).Volumes(&corev1.Volume{Name: dataDownloadName}).NodeName("node-1").Labels(map[string]string{
+				velerov1api.DataDownloadLabel: dataDownloadName,
+			}).Result(),
+			du:                     dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Result(),
+			prepareddDataDownloads: []string{dataDownloadName},
+		},
+		// Test case 4: Process Accepted InProgress DataDownload
+		{
+			name: "InProgressDataDownload",
+			pod: builder.ForPod(velerov1api.DefaultNamespace, dataDownloadName).Volumes(&corev1.Volume{Name: dataDownloadName}).NodeName("node-1").Labels(map[string]string{
+				velerov1api.DataDownloadLabel: dataDownloadName,
+			}).Result(),
+			du:                     dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Result(),
+			prepareddDataDownloads: []string{dataDownloadName},
+		},
+		// Test case 5: get resume error
+		{
+			name: "ResumeError",
+			pod: builder.ForPod(velerov1api.DefaultNamespace, dataDownloadName).Volumes(&corev1.Volume{Name: dataDownloadName}).NodeName("node-1").Labels(map[string]string{
+				velerov1api.DataDownloadLabel: dataDownloadName,
+			}).Result(),
+			needErrs:      []bool{false, false, false, false, false, true},
+			du:            dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Result(),
+			expectedError: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.TODO()
+			r, err := initDataDownloadReconciler(nil, test.needErrs...)
+			r.nodeName = "node-1"
+			require.NoError(t, err)
+			defer func() {
+				r.client.Delete(ctx, test.du, &kbclient.DeleteOptions{})
+				if test.pod != nil {
+					r.client.Delete(ctx, test.pod, &kbclient.DeleteOptions{})
+				}
+			}()
+
+			assert.NoError(t, r.client.Create(ctx, test.du))
+			if test.pod != nil {
+				assert.NoError(t, r.client.Create(ctx, test.pod))
+			}
+			// Run the test
+			err = r.AttemptDataDownloadResume(ctx, r.client, r.logger.WithField("name", test.name), test.du.Namespace)
+
+			if test.expectedError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+
+				// Verify DataDownload marked as Cancelled
+				for _, duName := range test.cancelledDataDownloads {
+					dataUpload := &velerov2alpha1api.DataDownload{}
+					err := r.client.Get(context.Background(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload)
+					require.NoError(t, err)
+					assert.Equal(t, velerov2alpha1api.DataDownloadPhaseCanceled, dataUpload.Status.Phase)
+				}
+				// Verify DataDownload marked as Accepted
+				for _, duName := range test.acceptedDataDownloads {
+					dataUpload := &velerov2alpha1api.DataDownload{}
+					err := r.client.Get(context.Background(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload)
+					require.NoError(t, err)
+					assert.Equal(t, velerov2alpha1api.DataDownloadPhaseAccepted, dataUpload.Status.Phase)
+				}
+				// Verify DataDownload marked as Prepared
+				for _, duName := range test.prepareddDataDownloads {
+					dataUpload := &velerov2alpha1api.DataDownload{}
+					err := r.client.Get(context.Background(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload)
+					require.NoError(t, err)
+					assert.Equal(t, velerov2alpha1api.DataDownloadPhasePrepared, dataUpload.Status.Phase)
+				}
+			}
+		})
+	}
+}

pkg/controller/data_upload_controller.go

@@ -274,7 +274,6 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 				return r.errorOut(ctx, du, err, "error to create data path", log)
 			}
 		}
-
 		// Update status to InProgress
 		original := du.DeepCopy()
 		du.Status.Phase = velerov2alpha1api.DataUploadPhaseInProgress
@@ -344,7 +343,8 @@ func (r *DataUploadReconciler) runCancelableDataUpload(ctx context.Context, fsBa
 	tags := map[string]string{
 		velerov1api.AsyncOperationIDLabel: du.Labels[velerov1api.AsyncOperationIDLabel],
 	}
-	if err := fsBackup.StartBackup(path, fmt.Sprintf("%s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC), "", false, tags, *du.Spec.DataMoverConfig); err != nil {
+
+	if err := fsBackup.StartBackup(path, fmt.Sprintf("%s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC), "", false, tags, du.Spec.DataMoverConfig); err != nil {
 		return r.errorOut(ctx, du, err, "error starting data path backup", log)
 	}
 
@@ -581,7 +581,7 @@ func (r *DataUploadReconciler) findDataUploadForPod(podObj client.Object) []reco
 	return []reconcile.Request{requests}
 }
 
-func (r *DataUploadReconciler) FindDataUploads(ctx context.Context, cli client.Client, ns string) ([]velerov2alpha1api.DataUpload, error) {
+func (r *DataUploadReconciler) FindDataUploadsByPod(ctx context.Context, cli client.Client, ns string) ([]velerov2alpha1api.DataUpload, error) {
 	pods := &corev1.PodList{}
 	var dataUploads []velerov2alpha1api.DataUpload
 	if err := cli.List(ctx, pods, &client.ListOptions{Namespace: ns}); err != nil {
@@ -605,6 +605,51 @@ func (r *DataUploadReconciler) FindDataUploads(ctx context.Context, cli client.C
 	return dataUploads, nil
 }
 
+func (r *DataUploadReconciler) findAcceptDataUploadsByNodeLabel(ctx context.Context, cli client.Client, ns string) ([]velerov2alpha1api.DataUpload, error) {
+	dataUploads := &velerov2alpha1api.DataUploadList{}
+	if err := cli.List(ctx, dataUploads, &client.ListOptions{Namespace: ns}); err != nil {
+		r.logger.WithError(errors.WithStack(err)).Error("failed to list datauploads")
+		return nil, errors.Wrapf(err, "failed to list datauploads")
+	}
+
+	var result []velerov2alpha1api.DataUpload
+	for _, du := range dataUploads.Items {
+		if du.Status.Phase != velerov2alpha1api.DataUploadPhaseAccepted {
+			continue
+		}
+		if du.Labels[acceptNodeLabelKey] == r.nodeName {
+			result = append(result, du)
+		}
+	}
+	return result, nil
+}
+
+func (r *DataUploadReconciler) CancelAcceptedDataupload(ctx context.Context, cli client.Client, ns string) {
+	r.logger.Infof("Reset accepted dataupload for node %s", r.nodeName)
+	dataUploads, err := r.findAcceptDataUploadsByNodeLabel(ctx, cli, ns)
+	if err != nil {
+		r.logger.WithError(err).Error("failed to find dataupload")
+		return
+	}
+
+	for _, du := range dataUploads {
+		if du.Spec.Cancel {
+			continue
+		}
+		err = UpdateDataUploadWithRetry(ctx, cli, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, r.logger.WithField("dataupload", du.Name),
+			func(dataUpload *velerov2alpha1api.DataUpload) {
+				dataUpload.Spec.Cancel = true
+				dataUpload.Status.Message = fmt.Sprintf("found a dataupload with status %q during the node-agent starting, mark it as cancel", du.Status.Phase)
+			})
+
+		r.logger.WithField("dataupload", du.GetName()).Warn(du.Status.Message)
+		if err != nil {
+			r.logger.WithError(errors.WithStack(err)).Errorf("failed to mark dataupload %q cancel", du.GetName())
+			continue
+		}
+	}
+}
+
 func (r *DataUploadReconciler) prepareDataUpload(du *velerov2alpha1api.DataUpload) {
 	du.Status.Phase = velerov2alpha1api.DataUploadPhasePrepared
 	du.Status.Node = r.nodeName
@@ -833,3 +878,34 @@ func UpdateDataUploadWithRetry(ctx context.Context, client client.Client, namesp
 		return true, nil
 	})
 }
+
+func (r *DataUploadReconciler) AttemptDataUploadResume(ctx context.Context, cli client.Client, logger *logrus.Entry, ns string) error {
+	if dataUploads, err := r.FindDataUploadsByPod(ctx, cli, ns); err != nil {
+		return errors.Wrap(err, "failed to find data uploads")
+	} else {
+		for _, du := range dataUploads {
+			if du.Status.Phase == velerov2alpha1api.DataUploadPhasePrepared {
+				// keep doing nothing let controller re-download the data
+				// the Prepared CR could be still handled by dataupload controller after node-agent restart
+				logger.WithField("dataupload", du.GetName()).Debug("find a dataupload with status prepared")
+			} else if du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress {
+				err = UpdateDataUploadWithRetry(ctx, cli, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, logger.WithField("dataupload", du.Name),
+					func(dataUpload *velerov2alpha1api.DataUpload) {
+						dataUpload.Spec.Cancel = true
+						dataUpload.Status.Message = fmt.Sprintf("found a dataupload with status %q during the node-agent starting, mark it as cancel", du.Status.Phase)
+					})
+
+				if err != nil {
+					logger.WithError(errors.WithStack(err)).Errorf("failed to mark dataupload %q into canceled", du.GetName())
+					continue
+				}
+				logger.WithField("dataupload", du.GetName()).Debug("mark dataupload into canceled")
+			}
+		}
+	}
+
+	//If the data upload is in Accepted status, the volume snapshot may be deleted and the exposed pod may not be created
+	// so we need to mark the data upload as canceled for it may not be recoverable
+	r.CancelAcceptedDataupload(ctx, cli, ns)
+	return nil
+}

pkg/controller/data_upload_controller_test.go

@@ -68,6 +68,7 @@ type FakeClient struct {
 	updateError    error
 	patchError     error
 	updateConflict error
+	listError      error
 }
 
 func (c *FakeClient) Get(ctx context.Context, key kbclient.ObjectKey, obj kbclient.Object) error {
@@ -106,8 +107,16 @@ func (c *FakeClient) Patch(ctx context.Context, obj kbclient.Object, patch kbcli
 	return c.Client.Patch(ctx, obj, patch, opts...)
 }
 
+func (c *FakeClient) List(ctx context.Context, list kbclient.ObjectList, opts ...kbclient.ListOption) error {
+	if c.listError != nil {
+		return c.listError
+	}
+
+	return c.Client.List(ctx, list, opts...)
+}
+
 func initDataUploaderReconciler(needError ...bool) (*DataUploadReconciler, error) {
-	var errs []error = make([]error, 5)
+	var errs []error = make([]error, 6)
 	for k, isError := range needError {
 		if k == 0 && isError {
 			errs[0] = fmt.Errorf("Get error")
@@ -118,7 +127,9 @@ func initDataUploaderReconciler(needError ...bool) (*DataUploadReconciler, error
 		} else if k == 3 && isError {
 			errs[3] = fmt.Errorf("Patch error")
 		} else if k == 4 && isError {
-			errs[4] = apierrors.NewConflict(velerov2alpha1api.Resource("datadownload"), dataDownloadName, errors.New("conflict"))
+			errs[4] = apierrors.NewConflict(velerov2alpha1api.Resource("dataupload"), dataUploadName, errors.New("conflict"))
+		} else if k == 5 && isError {
+			errs[5] = fmt.Errorf("List error")
 		}
 	}
 
@@ -198,6 +209,8 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci
 			fakeClient.patchError = needError[3]
 		} else if k == 4 {
 			fakeClient.updateConflict = needError[4]
+		} else if k == 5 {
+			fakeClient.listError = needError[5]
 		}
 	}
 
@@ -283,7 +296,7 @@ func (f *fakeDataUploadFSBR) Init(ctx context.Context, bslName string, sourceNam
 	return nil
 }
 
-func (f *fakeDataUploadFSBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfigs map[string]string) error {
+func (f *fakeDataUploadFSBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfigs *map[string]string) error {
 	du := f.du
 	original := f.du.DeepCopy()
 	du.Status.Phase = velerov2alpha1api.DataUploadPhaseCompleted
@@ -293,7 +306,7 @@ func (f *fakeDataUploadFSBR) StartBackup(source datapath.AccessPoint, realSource
 	return nil
 }
 
-func (f *fakeDataUploadFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error {
+func (f *fakeDataUploadFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs *map[string]string) error {
 	return nil
 }
 
@@ -983,7 +996,7 @@ func TestFindDataUploads(t *testing.T) {
 			require.NoError(t, err)
 			err = r.client.Create(ctx, &test.pod)
 			require.NoError(t, err)
-			uploads, err := r.FindDataUploads(context.Background(), r.client, "velero")
+			uploads, err := r.FindDataUploadsByPod(context.Background(), r.client, "velero")
 
 			if test.expectedError {
 				assert.Error(t, err)
@@ -994,3 +1007,110 @@ func TestFindDataUploads(t *testing.T) {
 		})
 	}
 }
+func TestAttemptDataUploadResume(t *testing.T) {
+	tests := []struct {
+		name                 string
+		dataUploads          []velerov2alpha1api.DataUpload
+		du                   *velerov2alpha1api.DataUpload
+		pod                  *corev1.Pod
+		needErrs             []bool
+		acceptedDataUploads  []string
+		prepareddDataUploads []string
+		cancelledDataUploads []string
+		expectedError        bool
+	}{
+		// Test case 1: Process Accepted DataUpload
+		{
+			name: "AcceptedDataUpload",
+			pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).NodeName("node-1").Labels(map[string]string{
+				velerov1api.DataUploadLabel: dataUploadName,
+			}).Result(),
+			du:                  dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Result(),
+			acceptedDataUploads: []string{dataUploadName},
+			expectedError:       false,
+		},
+		// Test case 2: Cancel an Accepted DataUpload
+		{
+			name: "CancelAcceptedDataUpload",
+			du:   dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Result(),
+		},
+		// Test case 3: Process Accepted Prepared DataUpload
+		{
+			name: "PreparedDataUpload",
+			pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).NodeName("node-1").Labels(map[string]string{
+				velerov1api.DataUploadLabel: dataUploadName,
+			}).Result(),
+			du:                   dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).Result(),
+			prepareddDataUploads: []string{dataUploadName},
+		},
+		// Test case 4: Process Accepted InProgress DataUpload
+		{
+			name: "InProgressDataUpload",
+			pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).NodeName("node-1").Labels(map[string]string{
+				velerov1api.DataUploadLabel: dataUploadName,
+			}).Result(),
+			du:                   dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).Result(),
+			prepareddDataUploads: []string{dataUploadName},
+		},
+		// Test case 5: get resume error
+		{
+			name: "ResumeError",
+			pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).NodeName("node-1").Labels(map[string]string{
+				velerov1api.DataUploadLabel: dataUploadName,
+			}).Result(),
+			needErrs:      []bool{false, false, false, false, false, true},
+			du:            dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).Result(),
+			expectedError: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.TODO()
+			r, err := initDataUploaderReconciler(test.needErrs...)
+			r.nodeName = "node-1"
+			require.NoError(t, err)
+			defer func() {
+				r.client.Delete(ctx, test.du, &kbclient.DeleteOptions{})
+				if test.pod != nil {
+					r.client.Delete(ctx, test.pod, &kbclient.DeleteOptions{})
+				}
+			}()
+
+			assert.NoError(t, r.client.Create(ctx, test.du))
+			if test.pod != nil {
+				assert.NoError(t, r.client.Create(ctx, test.pod))
+			}
+			// Run the test
+			err = r.AttemptDataUploadResume(ctx, r.client, r.logger.WithField("name", test.name), test.du.Namespace)
+
+			if test.expectedError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+
+				// Verify DataUploads marked as Cancelled
+				for _, duName := range test.cancelledDataUploads {
+					dataUpload := &velerov2alpha1api.DataUpload{}
+					err := r.client.Get(context.Background(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload)
+					require.NoError(t, err)
+					assert.Equal(t, velerov2alpha1api.DataUploadPhaseCanceled, dataUpload.Status.Phase)
+				}
+				// Verify DataUploads marked as Accepted
+				for _, duName := range test.acceptedDataUploads {
+					dataUpload := &velerov2alpha1api.DataUpload{}
+					err := r.client.Get(context.Background(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload)
+					require.NoError(t, err)
+					assert.Equal(t, velerov2alpha1api.DataUploadPhaseAccepted, dataUpload.Status.Phase)
+				}
+				// Verify DataUploads marked as Prepared
+				for _, duName := range test.prepareddDataUploads {
+					dataUpload := &velerov2alpha1api.DataUpload{}
+					err := r.client.Get(context.Background(), types.NamespacedName{Namespace: "velero", Name: duName}, dataUpload)
+					require.NoError(t, err)
+					assert.Equal(t, velerov2alpha1api.DataUploadPhasePrepared, dataUpload.Status.Phase)
+				}
+			}
+		})
+	}
+}

pkg/controller/pod_volume_backup_controller_test.go

@@ -103,7 +103,7 @@ func (b *fakeFSBR) Init(ctx context.Context, bslName string, sourceNamespace str
 	return nil
 }
 
-func (b *fakeFSBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfigs map[string]string) error {
+func (b *fakeFSBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfigs *map[string]string) error {
 	pvb := b.pvb
 
 	original := b.pvb.DeepCopy()
@@ -115,7 +115,7 @@ func (b *fakeFSBR) StartBackup(source datapath.AccessPoint, realSource string, p
 	return nil
 }
 
-func (b *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error {
+func (b *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs *map[string]string) error {
 	return nil
 }
 

pkg/datapath/file_system.go

@@ -129,7 +129,7 @@ func (fs *fileSystemBR) Close(ctx context.Context) {
 	fs.log.WithField("user", fs.jobName).Info("FileSystemBR is closed")
 }
 
-func (fs *fileSystemBR) StartBackup(source AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfig map[string]string) error {
+func (fs *fileSystemBR) StartBackup(source AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfig *map[string]string) error {
 	if !fs.initialized {
 		return errors.New("file system data path is not initialized")
 	}
@@ -150,7 +150,7 @@ func (fs *fileSystemBR) StartBackup(source AccessPoint, realSource string, paren
 	return nil
 }
 
-func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error {
+func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs *map[string]string) error {
 	if !fs.initialized {
 		return errors.New("file system data path is not initialized")
 	}

pkg/datapath/file_system_test.go

@@ -100,7 +100,7 @@ func TestAsyncBackup(t *testing.T) {
 			fs.initialized = true
 			fs.callbacks = test.callbacks
 
-			err := fs.StartBackup(AccessPoint{ByPath: test.path}, "", "", false, nil, make(map[string]string))
+			err := fs.StartBackup(AccessPoint{ByPath: test.path}, "", "", false, nil, &map[string]string{})
 			require.Equal(t, nil, err)
 
 			<-finish
@@ -183,7 +183,7 @@ func TestAsyncRestore(t *testing.T) {
 			fs.initialized = true
 			fs.callbacks = test.callbacks
 
-			err := fs.StartRestore(test.snapshot, AccessPoint{ByPath: test.path}, make(map[string]string))
+			err := fs.StartRestore(test.snapshot, AccessPoint{ByPath: test.path}, &map[string]string{})
 			require.Equal(t, nil, err)
 
 			<-finish

pkg/datapath/mocks/types.go

@@ -43,11 +43,11 @@ func (_m *AsyncBR) Init(ctx context.Context, bslName string, sourceNamespace str
 }
 
 // StartBackup provides a mock function with given fields: source, realSource, parentSnapshot, forceFull, tags, dataMoverConfig
-func (_m *AsyncBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, dataMoverConfig map[string]string) error {
+func (_m *AsyncBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, dataMoverConfig *map[string]string) error {
 	ret := _m.Called(source, realSource, parentSnapshot, forceFull, tags, dataMoverConfig)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(datapath.AccessPoint, string, string, bool, map[string]string, map[string]string) error); ok {
+	if rf, ok := ret.Get(0).(func(datapath.AccessPoint, string, string, bool, map[string]string, *map[string]string) error); ok {
 		r0 = rf(source, realSource, parentSnapshot, forceFull, tags, dataMoverConfig)
 	} else {
 		r0 = ret.Error(0)
@@ -56,13 +56,13 @@ func (_m *AsyncBR) StartBackup(source datapath.AccessPoint, realSource string, p
 	return r0
 }
 
-// StartRestore provides a mock function with given fields: snapshotID, target, uploaderConfigs
-func (_m *AsyncBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error {
-	ret := _m.Called(snapshotID, target, uploaderConfigs)
+// StartRestore provides a mock function with given fields: snapshotID, target, dataMoverConfig
+func (_m *AsyncBR) StartRestore(snapshotID string, target datapath.AccessPoint, dataMoverConfig *map[string]string) error {
+	ret := _m.Called(snapshotID, target, dataMoverConfig)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(string, datapath.AccessPoint, map[string]string) error); ok {
-		r0 = rf(snapshotID, target, uploaderConfigs)
+	if rf, ok := ret.Get(0).(func(string, datapath.AccessPoint, *map[string]string) error); ok {
+		r0 = rf(snapshotID, target, dataMoverConfig)
 	} else {
 		r0 = ret.Error(0)
 	}

pkg/datapath/types.go

@@ -62,10 +62,10 @@ type AsyncBR interface {
 	Init(ctx context.Context, bslName string, sourceNamespace string, uploaderType string, repositoryType string, repoIdentifier string, repositoryEnsurer *repository.Ensurer, credentialGetter *credentials.CredentialGetter) error
 
 	// StartBackup starts an asynchronous data path instance for backup
-	StartBackup(source AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, dataMoverConfig map[string]string) error
+	StartBackup(source AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, dataMoverConfig *map[string]string) error
 
 	// StartRestore starts an asynchronous data path instance for restore
-	StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error
+	StartRestore(snapshotID string, target AccessPoint, dataMoverConfig *map[string]string) error
 
 	// Cancel cancels an asynchronous data path instance
 	Cancel()

pkg/install/deployment.go

@@ -41,6 +41,7 @@ type podTemplateConfig struct {
 	withSecret                      bool
 	defaultRepoMaintenanceFrequency time.Duration
 	garbageCollectionFrequency      time.Duration
+	podVolumeOperationTimeout       time.Duration
 	plugins                         []string
 	features                        []string
 	defaultVolumesToFsBackup        bool
@@ -115,6 +116,12 @@ func WithGarbageCollectionFrequency(val time.Duration) podTemplateOption {
 	}
 }
 
+func WithPodVolumeOperationTimeout(val time.Duration) podTemplateOption {
+	return func(c *podTemplateConfig) {
+		c.podVolumeOperationTimeout = val
+	}
+}
+
 func WithPlugins(plugins []string) podTemplateOption {
 	return func(c *podTemplateConfig) {
 		c.plugins = plugins
@@ -212,6 +219,10 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment
 		args = append(args, fmt.Sprintf("--garbage-collection-frequency=%v", c.garbageCollectionFrequency))
 	}
 
+	if c.podVolumeOperationTimeout > 0 {
+		args = append(args, fmt.Sprintf("--fs-backup-timeout=%v", c.podVolumeOperationTimeout))
+	}
+
 	deployment := &appsv1.Deployment{
 		ObjectMeta: objectMeta(namespace, "velero"),
 		TypeMeta: metav1.TypeMeta{

pkg/install/resources.go

@@ -246,6 +246,7 @@ type VeleroOptions struct {
 	VSLConfig                       map[string]string
 	DefaultRepoMaintenanceFrequency time.Duration
 	GarbageCollectionFrequency      time.Duration
+	PodVolumeOperationTimeout       time.Duration
 	Plugins                         []string
 	NoDefaultBackupLocation         bool
 	CACertData                      []byte
@@ -335,6 +336,7 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList {
 		WithDefaultRepoMaintenanceFrequency(o.DefaultRepoMaintenanceFrequency),
 		WithServiceAccountName(serviceAccountName),
 		WithGarbageCollectionFrequency(o.GarbageCollectionFrequency),
+		WithPodVolumeOperationTimeout(o.PodVolumeOperationTimeout),
 		WithUploaderType(o.UploaderType),
 	}
 

pkg/podvolume/backupper.go

@@ -301,11 +301,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.
 			}
 		}
 
-		volumeBackup, err := newPodVolumeBackup(backup, pod, volume, repoIdentifier, b.uploaderType, pvc)
-		if err != nil {
-			errs = append(errs, errors.Wrapf(err, "error creating PodVolumeBackup for volume %s", volumeName))
-			continue
-		}
+		volumeBackup := newPodVolumeBackup(backup, pod, volume, repoIdentifier, b.uploaderType, pvc)
 		if err := veleroclient.CreateRetryGenerateName(b.crClient, b.ctx, volumeBackup); err != nil {
 			errs = append(errs, err)
 			continue
@@ -358,7 +354,7 @@ func isHostPathVolume(volume *corev1api.Volume, pvc *corev1api.PersistentVolumeC
 	return pv.Spec.HostPath != nil, nil
 }
 
-func newPodVolumeBackup(backup *velerov1api.Backup, pod *corev1api.Pod, volume corev1api.Volume, repoIdentifier, uploaderType string, pvc *corev1api.PersistentVolumeClaim) (*velerov1api.PodVolumeBackup, error) {
+func newPodVolumeBackup(backup *velerov1api.Backup, pod *corev1api.Pod, volume corev1api.Volume, repoIdentifier, uploaderType string, pvc *corev1api.PersistentVolumeClaim) *velerov1api.PodVolumeBackup {
 	pvb := &velerov1api.PodVolumeBackup{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace:    backup.Namespace,
@@ -415,15 +411,9 @@ func newPodVolumeBackup(backup *velerov1api.Backup, pod *corev1api.Pod, volume c
 		pvb.Spec.Tags["pvc-uid"] = string(pvc.UID)
 	}
 
-	if backup.Spec.BackupConfig != nil {
-		configJSON, err := uploaderconfig.MarshalToPVBConfig(backup.Spec.BackupConfig)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to marshal backup config")
-		}
-		pvb.Spec.UploaderSettings = map[string]string{
-			uploaderconfig.PodVolumeBackups: configJSON,
-		}
+	if backup.Spec.UploaderConfigForBackup != nil {
+		pvb.Spec.UploaderSettings = uploaderconfig.StoreBackupConfig(backup.Spec.UploaderConfigForBackup)
 	}
 
-	return pvb, nil
+	return pvb
 }

pkg/podvolume/restorer.go

@@ -177,12 +177,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData) []error {
 			}
 		}
 
-		volumeRestore, err := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc)
-		if err != nil {
-			errs = append(errs, errors.Wrapf(err, "error creating PodVolumeRestore for volume %s", volume))
-			continue
-		}
-
+		volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc)
 		if err := veleroclient.CreateRetryGenerateName(r.crClient, r.ctx, volumeRestore); err != nil {
 			errs = append(errs, errors.WithStack(err))
 			continue
@@ -251,7 +246,7 @@ ForEachVolume:
 	return errs
 }
 
-func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backupLocation, volume, snapshot, repoIdentifier, uploaderType, sourceNamespace string, pvc *corev1api.PersistentVolumeClaim) (*velerov1api.PodVolumeRestore, error) {
+func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backupLocation, volume, snapshot, repoIdentifier, uploaderType, sourceNamespace string, pvc *corev1api.PersistentVolumeClaim) *velerov1api.PodVolumeRestore {
 	pvr := &velerov1api.PodVolumeRestore{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace:    restore.Namespace,
@@ -291,18 +286,11 @@ func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backu
 		pvr.Labels[velerov1api.PVCUIDLabel] = string(pvc.UID)
 	}
 
-	if restore.Spec.RestoreConfig != nil {
-		configJSON, err := uploaderconfig.MarshalToPVRConfig(restore.Spec.RestoreConfig)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to marshal restore config")
-		}
-
-		pvr.Spec.UploaderSettings = map[string]string{
-			uploaderconfig.PodVolumeRestores: configJSON,
-		}
+	if restore.Spec.UploaderConfigForRestore != nil {
+		pvr.Spec.UploaderSettings = uploaderconfig.StoreRestoreConfig(restore.Spec.UploaderConfigForRestore)
 	}
 
-	return pvr, nil
+	return pvr
 }
 
 func getVolumesRepositoryType(volumes map[string]volumeBackupInfo) (string, error) {

pkg/restore/restore.go

@@ -325,6 +325,7 @@ func (kr *kubernetesRestorer) RestoreWithResolvers(
 		resourceModifiers:              req.ResourceModifiers,
 		disableInformerCache:           req.DisableInformerCache,
 		featureVerifier:                kr.featureVerifier,
+		hookTracker:                    hook.NewHookTracker(),
 	}
 
 	return restoreCtx.execute()
@@ -377,6 +378,7 @@ type restoreContext struct {
 	resourceModifiers              *resourcemodifiers.ResourceModifiers
 	disableInformerCache           bool
 	featureVerifier                features.Verifier
+	hookTracker                    *hook.HookTracker
 }
 
 type resourceClientKey struct {
@@ -544,6 +546,23 @@ func (ctx *restoreContext) execute() (results.Result, results.Result) {
 		errs.Merge(&e)
 	}
 
+	var createdOrUpdatedCRDs bool
+	for _, restoredItem := range ctx.restoredItems {
+		if restoredItem.action == itemRestoreResultCreated || restoredItem.action == itemRestoreResultUpdated {
+			createdOrUpdatedCRDs = true
+			break
+		}
+	}
+	// If we just restored custom resource definitions (CRDs), refresh
+	// discovery because the restored CRDs may have created or updated new APIs that
+	// didn't previously exist in the cluster, and we want to be able to
+	// resolve & restore instances of them in subsequent loop iterations.
+	if createdOrUpdatedCRDs {
+		if err := ctx.discoveryHelper.Refresh(); err != nil {
+			warnings.Add("", errors.Wrap(err, "refresh discovery after restoring CRDs"))
+		}
+	}
+
 	// Restore everything else
 	selectedResourceCollection, _, w, e := ctx.getOrderedResourceCollection(
 		backupResources,
@@ -629,11 +648,6 @@ func (ctx *restoreContext) execute() (results.Result, results.Result) {
 	updated.Status.Progress.TotalItems = len(ctx.restoredItems)
 	updated.Status.Progress.ItemsRestored = len(ctx.restoredItems)
 
-	err = kube.PatchResource(ctx.restore, updated, ctx.kbClient)
-	if err != nil {
-		ctx.log.WithError(errors.WithStack((err))).Warn("Updating restore status.progress")
-	}
-
 	// Wait for all of the pod volume restore goroutines to be done, which is
 	// only possible once all of their errors have been received by the loop
 	// below, then close the podVolumeErrs channel so the loop terminates.
@@ -668,6 +682,19 @@ func (ctx *restoreContext) execute() (results.Result, results.Result) {
 	}
 	ctx.log.Info("Done waiting for all post-restore exec hooks to complete")
 
+	// update hooks execution status
+	if updated.Status.HookStatus == nil {
+		updated.Status.HookStatus = &velerov1api.HookStatus{}
+	}
+	updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed = ctx.hookTracker.Stat()
+	ctx.log.Infof("hookTracker: %+v, hookAttempted: %d, hookFailed: %d", ctx.hookTracker.GetTracker(), updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed)
+
+	// patch the restore status
+	err = kube.PatchResource(ctx.restore, updated, ctx.kbClient)
+	if err != nil {
+		ctx.log.WithError(errors.WithStack((err))).Warn("Updating restore status")
+	}
+
 	return warnings, errs
 }
 
@@ -762,15 +789,6 @@ func (ctx *restoreContext) processSelectedResource(
 		}
 	}
 
-	// If we just restored custom resource definitions (CRDs), refresh
-	// discovery because the restored CRDs may have created new APIs that
-	// didn't previously exist in the cluster, and we want to be able to
-	// resolve & restore instances of them in subsequent loop iterations.
-	if groupResource == kuberesource.CustomResourceDefinitions {
-		if err := ctx.discoveryHelper.Refresh(); err != nil {
-			warnings.Add("", errors.Wrap(err, "refresh discovery after restoring CRDs"))
-		}
-	}
 	return processedItems, warnings, errs
 }
 
@@ -1963,6 +1981,7 @@ func (ctx *restoreContext) waitExec(createdObj *unstructured.Unstructured) {
 			ctx.resourceRestoreHooks,
 			pod,
 			ctx.log,
+			ctx.hookTracker,
 		)
 		if err != nil {
 			ctx.log.WithError(err).Errorf("error getting exec hooks for pod %s/%s", pod.Namespace, pod.Name)
@@ -1970,7 +1989,7 @@ func (ctx *restoreContext) waitExec(createdObj *unstructured.Unstructured) {
 			return
 		}
 
-		if errs := ctx.waitExecHookHandler.HandleHooks(ctx.hooksContext, ctx.log, pod, execHooksByContainer); len(errs) > 0 {
+		if errs := ctx.waitExecHookHandler.HandleHooks(ctx.hooksContext, ctx.log, pod, execHooksByContainer, ctx.hookTracker); len(errs) > 0 {
 			ctx.log.WithError(kubeerrs.NewAggregate(errs)).Error("unable to successfully execute post-restore hooks")
 			ctx.hooksCancelFunc()
 

pkg/uploader/kopia/snapshot.go

@@ -18,7 +18,6 @@ package kopia
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"math"
 	"os"
@@ -39,7 +38,7 @@ import (
 	"github.com/kopia/kopia/snapshot/snapshotfs"
 	"github.com/pkg/errors"
 
-	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/util/uploaderconfig"
 
 	"github.com/vmware-tanzu/velero/pkg/kopia"
 	"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
@@ -107,21 +106,18 @@ func getDefaultPolicy() *policy.Policy {
 	}
 }
 
-func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, uploaderCfg map[string]string) (*policy.Tree, error) {
+func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, uploaderCfg *map[string]string) (*policy.Tree, error) {
 	// some internal operations from Kopia code retrieves policies from repo directly, so we need to persist the policy to repo
-	backupCfg := velerov1api.BackupConfig{}
-	// currently, we only have one uploader config in one uploader config so we can just loop through it
-	for configItem, jsonConfig := range uploaderCfg {
-		err := json.Unmarshal([]byte(jsonConfig), &backupCfg)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to parse %s uploader config", configItem)
-		}
-		break
-	}
-
 	curPolicy := getDefaultPolicy()
-	if backupCfg.ParallelFilesUpload > 0 {
-		curPolicy.UploadPolicy.MaxParallelFileReads = newOptionalInt(backupCfg.ParallelFilesUpload)
+
+	if uploaderCfg != nil {
+		uploaderConfig, err := uploaderconfig.GetBackupConfig(uploaderCfg)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get uploader config")
+		}
+		if uploaderConfig.ParallelFilesUpload > 0 {
+			curPolicy.UploadPolicy.MaxParallelFileReads = newOptionalInt(uploaderConfig.ParallelFilesUpload)
+		}
 	}
 
 	err := setPolicyFunc(ctx, rep, sourceInfo, curPolicy)
@@ -145,7 +141,7 @@ func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snap
 
 // Backup backup specific sourcePath and update progress
 func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string,
-	forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+	forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 	if fsUploader == nil {
 		return nil, false, errors.New("get empty kopia uploader")
 	}
@@ -241,7 +237,7 @@ func SnapshotSource(
 	forceFull bool,
 	parentSnapshot string,
 	snapshotTags map[string]string,
-	uploaderCfg map[string]string,
+	uploaderCfg *map[string]string,
 	log logrus.FieldLogger,
 	description string,
 ) (string, int64, error) {
@@ -373,7 +369,7 @@ func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sour
 }
 
 // Restore restore specific sourcePath with given snapshotID and update progress
-func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string,
+func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string,
 	log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
 	log.Info("Start to restore...")
 
@@ -396,23 +392,20 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress,
 		return 0, 0, errors.Wrapf(err, "Unable to resolve path %v", dest)
 	}
 
-	restoreCfg := velerov1api.RestoreConfig{}
-	// currently, we only have one uploader config in map so we can just loop through it
-	for configItem, jsonConfig := range uploaderCfg {
-		err := json.Unmarshal([]byte(jsonConfig), &restoreCfg)
-		if err != nil {
-			return 0, 0, errors.Wrapf(err, "failed to parse %s uploader config", configItem)
-		}
-		break
-	}
-
 	fsOutput := &restore.FilesystemOutput{
 		TargetPath:             path,
 		OverwriteDirectories:   true,
 		OverwriteFiles:         true,
 		OverwriteSymlinks:      true,
 		IgnorePermissionErrors: true,
-		WriteSparseFiles:       restoreCfg.WriteSparseFiles,
+	}
+
+	if uploaderCfg != nil {
+		restoreCfg, err := uploaderconfig.GetRestoreConfig(uploaderCfg)
+		if err != nil {
+			return 0, 0, errors.Wrap(err, "failed to get uploader config")
+		}
+		fsOutput.WriteSparseFiles = restoreCfg.WriteSparseFiles
 	}
 	log.Debugf("Restore filesystem output %v", fsOutput)
 

pkg/uploader/kopia/snapshot_test.go

@@ -96,7 +96,7 @@ func TestSnapshotSource(t *testing.T) {
 	testCases := []struct {
 		name        string
 		args        []mockArgs
-		uploaderCfg map[string]string
+		uploaderCfg *map[string]string
 		notError    bool
 	}{
 		{
@@ -152,7 +152,7 @@ func TestSnapshotSource(t *testing.T) {
 			notError: false,
 		},
 		{
-			name: "set policy with ParallelFilesUpload",
+			name: "set policy with parallel files upload",
 			args: []mockArgs{
 				{methodName: "LoadSnapshot", returns: []interface{}{manifest, nil}},
 				{methodName: "SaveSnapshot", returns: []interface{}{manifest.ID, nil}},
@@ -162,8 +162,10 @@ func TestSnapshotSource(t *testing.T) {
 				{methodName: "Upload", returns: []interface{}{manifest, nil}},
 				{methodName: "Flush", returns: []interface{}{nil}},
 			},
-			uploaderCfg: map[string]string{"ParallelFilesUpload": "10"},
-			notError:    true,
+			uploaderCfg: &map[string]string{
+				"ParallelFilesUpload": "10",
+			},
+			notError: true,
 		},
 		{
 			name: "failed to upload snapshot",
@@ -645,9 +647,9 @@ func TestBackup(t *testing.T) {
 			var snapshotInfo *uploader.SnapshotInfo
 			var err error
 			if tc.isEmptyUploader {
-				snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), nil, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, make(map[string]string), tc.tags, &logrus.Logger{})
+				snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), nil, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, &map[string]string{}, tc.tags, &logrus.Logger{})
 			} else {
-				snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), s.uploderMock, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, make(map[string]string), tc.tags, &logrus.Logger{})
+				snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), s.uploderMock, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, &map[string]string{}, tc.tags, &logrus.Logger{})
 			}
 			// Check if the returned error matches the expected error
 			if tc.expectedError != nil {
@@ -786,7 +788,7 @@ func TestRestore(t *testing.T) {
 			repoWriterMock.On("OpenObject", mock.Anything, mock.Anything).Return(em, nil)
 
 			progress := new(Progress)
-			bytesRestored, fileCount, err := Restore(context.Background(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, make(map[string]string), logrus.New(), nil)
+			bytesRestored, fileCount, err := Restore(context.Background(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, &map[string]string{}, logrus.New(), nil)
 
 			// Check if the returned error matches the expected error
 			if tc.expectedError != nil {

pkg/uploader/provider/kopia.go

@@ -119,7 +119,7 @@ func (kp *kopiaProvider) RunBackup(
 	forceFull bool,
 	parentSnapshot string,
 	volMode uploader.PersistentVolumeMode,
-	uploaderCfg map[string]string,
+	uploaderCfg *map[string]string,
 	updater uploader.ProgressUpdater) (string, bool, error) {
 	if updater == nil {
 		return "", false, errors.New("Need to initial backup progress updater first")
@@ -204,7 +204,7 @@ func (kp *kopiaProvider) RunRestore(
 	snapshotID string,
 	volumePath string,
 	volMode uploader.PersistentVolumeMode,
-	uploaderCfg map[string]string,
+	uploaderCfg *map[string]string,
 	updater uploader.ProgressUpdater) error {
 	log := kp.log.WithFields(logrus.Fields{
 		"snapshotID": snapshotID,

pkg/uploader/provider/kopia_test.go

@@ -68,34 +68,34 @@ func TestRunBackup(t *testing.T) {
 
 	testCases := []struct {
 		name           string
-		hookBackupFunc func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error)
+		hookBackupFunc func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error)
 		volMode        uploader.PersistentVolumeMode
 		notError       bool
 	}{
 		{
 			name: "success to backup",
-			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 				return &uploader.SnapshotInfo{}, false, nil
 			},
 			notError: true,
 		},
 		{
 			name: "get error to backup",
-			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 				return &uploader.SnapshotInfo{}, false, errors.New("failed to backup")
 			},
 			notError: false,
 		},
 		{
 			name: "got empty snapshot",
-			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 				return nil, true, errors.New("snapshot is empty")
 			},
 			notError: false,
 		},
 		{
 			name: "success to backup block mode volume",
-			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 				return &uploader.SnapshotInfo{}, false, nil
 			},
 			volMode:  uploader.PersistentVolumeBlock,
@@ -108,7 +108,7 @@ func TestRunBackup(t *testing.T) {
 				tc.volMode = uploader.PersistentVolumeFilesystem
 			}
 			BackupFunc = tc.hookBackupFunc
-			_, _, err := kp.RunBackup(context.Background(), "var", "", nil, false, "", tc.volMode, map[string]string{}, &updater)
+			_, _, err := kp.RunBackup(context.Background(), "var", "", nil, false, "", tc.volMode, &map[string]string{}, &updater)
 			if tc.notError {
 				assert.NoError(t, err)
 			} else {
@@ -125,27 +125,27 @@ func TestRunRestore(t *testing.T) {
 
 	testCases := []struct {
 		name            string
-		hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error)
+		hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error)
 		notError        bool
 		volMode         uploader.PersistentVolumeMode
 	}{
 		{
 			name: "normal restore",
-			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
+			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
 				return 0, 0, nil
 			},
 			notError: true,
 		},
 		{
 			name: "failed to restore",
-			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
+			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
 				return 0, 0, errors.New("failed to restore")
 			},
 			notError: false,
 		},
 		{
 			name: "normal block mode restore",
-			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
+			hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
 				return 0, 0, nil
 			},
 			volMode:  uploader.PersistentVolumeBlock,
@@ -159,7 +159,7 @@ func TestRunRestore(t *testing.T) {
 				tc.volMode = uploader.PersistentVolumeFilesystem
 			}
 			RestoreFunc = tc.hookRestoreFunc
-			err := kp.RunRestore(context.Background(), "", "/var", tc.volMode, map[string]string{}, &updater)
+			err := kp.RunRestore(context.Background(), "", "/var", tc.volMode, &map[string]string{}, &updater)
 			if tc.notError {
 				assert.NoError(t, err)
 			} else {

pkg/uploader/provider/mocks/Provider.go

@@ -30,28 +30,28 @@ func (_m *Provider) Close(ctx context.Context) error {
 }
 
 // RunBackup provides a mock function with given fields: ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater
-func (_m *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, error) {
+func (_m *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg *map[string]string, updater uploader.ProgressUpdater) (string, bool, error) {
 	ret := _m.Called(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 
 	var r0 string
 	var r1 bool
 	var r2 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (string, bool, error)); ok {
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) (string, bool, error)); ok {
 		return rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) string); ok {
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) string); ok {
 		r0 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 	} else {
 		r0 = ret.Get(0).(string)
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) bool); ok {
+	if rf, ok := ret.Get(1).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) bool); ok {
 		r1 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 	} else {
 		r1 = ret.Get(1).(bool)
 	}
 
-	if rf, ok := ret.Get(2).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
+	if rf, ok := ret.Get(2).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) error); ok {
 		r2 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 	} else {
 		r2 = ret.Error(2)
@@ -61,11 +61,11 @@ func (_m *Provider) RunBackup(ctx context.Context, path string, realSource strin
 }
 
 // RunRestore provides a mock function with given fields: ctx, snapshotID, volumePath, volMode, uploaderConfig, updater
-func (_m *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) error {
+func (_m *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig *map[string]string, updater uploader.ProgressUpdater) error {
 	ret := _m.Called(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok {
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, *map[string]string, uploader.ProgressUpdater) error); ok {
 		r0 = rf(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)
 	} else {
 		r0 = ret.Error(0)

pkg/uploader/provider/provider.go

@@ -49,7 +49,7 @@ type Provider interface {
 		forceFull bool,
 		parentSnapshot string,
 		volMode uploader.PersistentVolumeMode,
-		uploaderCfg map[string]string,
+		uploaderCfg *map[string]string,
 		updater uploader.ProgressUpdater) (string, bool, error)
 	// RunRestore which will do restore for one specific volume with given snapshot id and return error
 	// updater is used for updating backup progress which implement by third-party
@@ -58,7 +58,7 @@ type Provider interface {
 		snapshotID string,
 		volumePath string,
 		volMode uploader.PersistentVolumeMode,
-		uploaderConfig map[string]string,
+		uploaderConfig *map[string]string,
 		updater uploader.ProgressUpdater) error
 	// Close which will close related repository
 	Close(ctx context.Context) error

pkg/uploader/provider/restic.go

@@ -18,7 +18,6 @@ package provider
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
 	"strings"
@@ -32,6 +31,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/restic"
 	"github.com/vmware-tanzu/velero/pkg/uploader"
 	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
+	"github.com/vmware-tanzu/velero/pkg/util/uploaderconfig"
 )
 
 // resticBackupCMDFunc and resticRestoreCMDFunc are mainly used to make testing more convenient
@@ -123,7 +123,7 @@ func (rp *resticProvider) RunBackup(
 	forceFull bool,
 	parentSnapshot string,
 	volMode uploader.PersistentVolumeMode,
-	uploaderCfg map[string]string,
+	uploaderCfg *map[string]string,
 	updater uploader.ProgressUpdater) (string, bool, error) {
 	if updater == nil {
 		return "", false, errors.New("Need to initial backup progress updater first")
@@ -146,18 +146,15 @@ func (rp *resticProvider) RunBackup(
 		"parentSnapshot": parentSnapshot,
 	})
 
-	backupCfg := velerov1api.BackupConfig{}
-	// currently, we only have one uploader config in one uploader config so we can just loop through it
-	for configItem, jsonConfig := range uploaderCfg {
-		err := json.Unmarshal([]byte(jsonConfig), &backupCfg)
+	if uploaderCfg != nil {
+		uploaderConfig, err := uploaderconfig.GetBackupConfig(uploaderCfg)
 		if err != nil {
-			return "", false, errors.Wrapf(err, "failed to parse %s config", configItem)
+			return "", false, errors.Wrap(err, "failed to get uploader config")
 		}
-		break
-	}
 
-	if backupCfg.ParallelFilesUpload > 0 {
-		log.Warnf("ParallelFilesUpload is set to %d, but restic does not support parallel file uploads. Ignoring.", backupCfg.ParallelFilesUpload)
+		if uploaderConfig.ParallelFilesUpload > 0 {
+			log.Warnf("ParallelFilesUpload is set to %d, but restic does not support parallel file uploads. Ignoring.", uploaderConfig.ParallelFilesUpload)
+		}
 	}
 
 	backupCmd := resticBackupCMDFunc(rp.repoIdentifier, rp.credentialsFile, path, tags)
@@ -201,7 +198,7 @@ func (rp *resticProvider) RunRestore(
 	snapshotID string,
 	volumePath string,
 	volMode uploader.PersistentVolumeMode,
-	uploaderCfg map[string]string,
+	uploaderCfg *map[string]string,
 	updater uploader.ProgressUpdater) error {
 	if updater == nil {
 		return errors.New("Need to initial backup progress updater first")
@@ -222,11 +219,13 @@ func (rp *resticProvider) RunRestore(
 		restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, rp.extraFlags...)
 	}
 
-	extraFlags, err := rp.parseRestoreExtraFlags(uploaderCfg)
-	if err != nil {
-		return errors.Wrap(err, "failed to parse uploader config")
-	} else if len(extraFlags) != 0 {
-		restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, extraFlags...)
+	if uploaderCfg != nil {
+		extraFlags, err := rp.parseRestoreExtraFlags(uploaderCfg)
+		if err != nil {
+			return errors.Wrap(err, "failed to parse uploader config")
+		} else if len(extraFlags) != 0 {
+			restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, extraFlags...)
+		}
 	}
 
 	stdout, stderr, err := restic.RunRestore(restoreCmd, log, updater)
@@ -235,19 +234,14 @@ func (rp *resticProvider) RunRestore(
 	return err
 }
 
-func (rp *resticProvider) parseRestoreExtraFlags(uploaderCfg map[string]string) ([]string, error) {
+func (rp *resticProvider) parseRestoreExtraFlags(uploaderCfg *map[string]string) ([]string, error) {
 	extraFlags := []string{}
-	restoreCfg := velerov1api.RestoreConfig{}
-	// currently, we only have one uploader config in map so we can just loop through it
-	for configItem, jsonConfig := range uploaderCfg {
-		err := json.Unmarshal([]byte(jsonConfig), &restoreCfg)
-		if err != nil {
-			return extraFlags, errors.Wrapf(err, "failed to parse %s uploader config", configItem)
-		}
-		break
+	uploaderConfig, err := uploaderconfig.GetRestoreConfig(uploaderCfg)
+	if err != nil {
+		return extraFlags, errors.Wrap(err, "failed to get uploader config")
 	}
 
-	if restoreCfg.WriteSparseFiles {
+	if uploaderConfig.WriteSparseFiles {
 		extraFlags = append(extraFlags, "--sparse")
 	}
 	return extraFlags, nil

pkg/uploader/provider/restic_test.go

@@ -38,7 +38,6 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/uploader"
 	"github.com/vmware-tanzu/velero/pkg/util"
 	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
-	"github.com/vmware-tanzu/velero/pkg/util/uploaderconfig"
 )
 
 func TestResticRunBackup(t *testing.T) {
@@ -151,9 +150,9 @@ func TestResticRunBackup(t *testing.T) {
 			}
 			if !tc.nilUpdater {
 				updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()}
-				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, &updater)
+				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, &map[string]string{}, &updater)
 			} else {
-				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, nil)
+				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, &map[string]string{}, nil)
 			}
 
 			tc.rp.log.Infof("test name %v error %v", tc.name, err)
@@ -224,9 +223,9 @@ func TestResticRunRestore(t *testing.T) {
 			var err error
 			if !tc.nilUpdater {
 				updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()}
-				err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, map[string]string{}, &updater)
+				err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, &map[string]string{}, &updater)
 			} else {
-				err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, map[string]string{}, nil)
+				err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, &map[string]string{}, nil)
 			}
 
 			tc.rp.log.Infof("test name %v error %v", tc.name, err)
@@ -418,20 +417,20 @@ func TestParseUploaderConfig(t *testing.T) {
 
 	testCases := []struct {
 		name           string
-		uploaderConfig map[string]string
+		uploaderConfig *map[string]string
 		expectedFlags  []string
 	}{
 		{
 			name: "SparseFilesEnabled",
-			uploaderConfig: map[string]string{
-				uploaderconfig.PodVolumeRestores: `{"WriteSparseFiles": true}`,
+			uploaderConfig: &map[string]string{
+				"WriteSparseFiles": "true",
 			},
 			expectedFlags: []string{"--sparse"},
 		},
 		{
 			name: "SparseFilesDisabled",
-			uploaderConfig: map[string]string{
-				uploaderconfig.PodVolumeRestores: `{"WriteSparseFiles": false}`,
+			uploaderConfig: &map[string]string{
+				"writeSparseFiles": "false",
 			},
 			expectedFlags: []string{},
 		},

pkg/util/uploaderconfig/uploaderconfig.go

@@ -1,7 +1,7 @@
 package uploaderconfig
 
 import (
-	"encoding/json"
+	"strconv"
 
 	"github.com/pkg/errors"
 
@@ -9,74 +9,42 @@ import (
 )
 
 const (
-	PodVolumeBackups  = "PodVolumeBackups"
-	PodVolumeRestores = "PodVolumeRestores"
-	DataUploads       = "DataUploads"
-	DataDownloads     = "DataDownloads"
+	parallelFilesUpload = "ParallelFilesUpload"
+	writeSparseFiles    = "WriteSparseFiles"
 )
 
-type PVBConfig struct {
-	ParallelFilesUpload int `json:"parallelFilesUpload,omitempty"`
+func StoreBackupConfig(config *velerov1api.UploaderConfigForBackup) *map[string]string {
+	data := make(map[string]string)
+	data[parallelFilesUpload] = strconv.Itoa(config.ParallelFilesUpload)
+	return &data
 }
 
-type PVRConfig struct {
-	WriteSparseFiles bool `json:"writeSparseFiles,omitempty"`
+func StoreRestoreConfig(config *velerov1api.UploaderConfigForRestore) *map[string]string {
+	data := make(map[string]string)
+	data[writeSparseFiles] = strconv.FormatBool(config.WriteSparseFiles)
+	return &data
 }
 
-func MarshalToPVBConfig(backupConfig *velerov1api.BackupConfig) (string, error) {
-	jsonData, err := json.Marshal(backupConfig)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to marshal backup config")
-	}
-
-	var pvb PVBConfig
-	err = json.Unmarshal(jsonData, &pvb)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to unmarshal backup config")
-	}
-
-	finalJSONData, err := json.Marshal(pvb)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to marshal backup config")
-	}
-
-	return string(finalJSONData), nil
-}
-
-func MarshalToPVRConfig(restoreConfig *velerov1api.RestoreConfig) (string, error) {
-	jsonData, err := json.Marshal(restoreConfig)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to marshal restore config")
-	}
-
-	var pvr PVRConfig
-	err = json.Unmarshal(jsonData, &pvr)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to unmarshal restore config")
-	}
-
-	finalJSONData, err := json.Marshal(pvr)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to marshal restore config")
-	}
-
-	return string(finalJSONData), nil
-}
-
-func ParseBackupConfig(str string) (velerov1api.BackupConfig, error) {
-	var config velerov1api.BackupConfig
-	err := json.Unmarshal([]byte(str), &config)
-	if err != nil {
-		return velerov1api.BackupConfig{}, err
+func GetBackupConfig(data *map[string]string) (velerov1api.UploaderConfigForBackup, error) {
+	config := velerov1api.UploaderConfigForBackup{}
+	var err error
+	if item, ok := (*data)[parallelFilesUpload]; ok {
+		config.ParallelFilesUpload, err = strconv.Atoi(item)
+		if err != nil {
+			return velerov1api.UploaderConfigForBackup{}, errors.Wrap(err, "failed to parse ParallelFilesUpload")
+		}
 	}
 	return config, nil
 }
 
-func ParseRestoreConfig(str string) (velerov1api.RestoreConfig, error) {
-	var config velerov1api.RestoreConfig
-	err := json.Unmarshal([]byte(str), &config)
-	if err != nil {
-		return velerov1api.RestoreConfig{}, err
+func GetRestoreConfig(data *map[string]string) (velerov1api.UploaderConfigForRestore, error) {
+	config := velerov1api.UploaderConfigForRestore{}
+	var err error
+	if item, ok := (*data)[writeSparseFiles]; ok {
+		config.WriteSparseFiles, err = strconv.ParseBool(item)
+		if err != nil {
+			return velerov1api.UploaderConfigForRestore{}, errors.Wrap(err, "failed to parse WriteSparseFiles")
+		}
 	}
 	return config, nil
 }

pkg/util/uploaderconfig/uploaderconfig_test.go

@@ -0,0 +1,95 @@
+package uploaderconfig
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+)
+
+func TestStoreBackupConfig(t *testing.T) {
+	config := &velerov1api.UploaderConfigForBackup{
+		ParallelFilesUpload: 3,
+	}
+
+	expectedData := map[string]string{
+		parallelFilesUpload: "3",
+	}
+
+	result := StoreBackupConfig(config)
+
+	if !reflect.DeepEqual(*result, expectedData) {
+		t.Errorf("Expected: %v, but got: %v", expectedData, *result)
+	}
+}
+
+func TestStoreRestoreConfig(t *testing.T) {
+	config := &velerov1api.UploaderConfigForRestore{
+		WriteSparseFiles: true,
+	}
+
+	expectedData := map[string]string{
+		writeSparseFiles: "true",
+	}
+
+	result := StoreRestoreConfig(config)
+
+	if !reflect.DeepEqual(*result, expectedData) {
+		t.Errorf("Expected: %v, but got: %v", expectedData, *result)
+	}
+}
+
+func TestGetBackupConfig(t *testing.T) {
+	data := &map[string]string{
+		parallelFilesUpload: "3",
+	}
+
+	expectedConfig := velerov1api.UploaderConfigForBackup{
+		ParallelFilesUpload: 3,
+	}
+
+	result, err := GetBackupConfig(data)
+
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(result, expectedConfig) {
+		t.Errorf("Expected: %v, but got: %v", expectedConfig, result)
+	}
+
+	// Test error case
+	(*data)[parallelFilesUpload] = "invalid"
+	_, err = GetBackupConfig(data)
+	if !strings.Contains(err.Error(), "failed to parse ParallelFilesUpload") {
+		t.Errorf("Expected error message containing 'failed to parse ParallelFilesUpload', but got: %v", err)
+	}
+}
+
+func TestGetRestoreConfig(t *testing.T) {
+	data := &map[string]string{
+		writeSparseFiles: "true",
+	}
+
+	expectedConfig := velerov1api.UploaderConfigForRestore{
+		WriteSparseFiles: true,
+	}
+
+	result, err := GetRestoreConfig(data)
+
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(result, expectedConfig) {
+		t.Errorf("Expected: %v, but got: %v", expectedConfig, result)
+	}
+
+	// Test error case
+	(*data)[writeSparseFiles] = "invalid"
+	_, err = GetRestoreConfig(data)
+	if !strings.Contains(err.Error(), "failed to parse WriteSparseFiles") {
+		t.Errorf("Expected error message containing 'failed to parse WriteSparseFiles', but got: %v", err)
+	}
+}

site/content/docs/main/custom-plugins.md

@@ -37,7 +37,7 @@ When naming your plugin, keep in mind that the full name needs to conform to the
 - have two parts, prefix + name, separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same prefix + name cannot not already exist
+- a plugin with the same prefix + name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.0.0/plugins.md

@@ -15,7 +15,7 @@ When naming your plugin, keep in mind that the name needs to conform to these ru
 - have two parts separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same name cannot not already exist
+- a plugin with the same name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.1.0/plugins.md

@@ -15,7 +15,7 @@ When naming your plugin, keep in mind that the name needs to conform to these ru
 - have two parts separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same name cannot not already exist
+- a plugin with the same name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.10/custom-plugins.md

@@ -37,7 +37,7 @@ When naming your plugin, keep in mind that the full name needs to conform to the
 - have two parts, prefix + name, separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same prefix + name cannot not already exist
+- a plugin with the same prefix + name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.11/custom-plugins.md

@@ -37,7 +37,7 @@ When naming your plugin, keep in mind that the full name needs to conform to the
 - have two parts, prefix + name, separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same prefix + name cannot not already exist
+- a plugin with the same prefix + name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.2.0/custom-plugins.md

@@ -15,7 +15,7 @@ When naming your plugin, keep in mind that the name needs to conform to these ru
 - have two parts separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same name cannot not already exist
+- a plugin with the same name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.3.0/custom-plugins.md

@@ -15,7 +15,7 @@ When naming your plugin, keep in mind that the name needs to conform to these ru
 - have two parts separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same name cannot not already exist
+- a plugin with the same name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.3.1/custom-plugins.md

@@ -15,7 +15,7 @@ When naming your plugin, keep in mind that the name needs to conform to these ru
 - have two parts separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same name cannot not already exist
+- a plugin with the same name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.3.2/custom-plugins.md

@@ -15,7 +15,7 @@ When naming your plugin, keep in mind that the name needs to conform to these ru
 - have two parts separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same name cannot not already exist
+- a plugin with the same name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.4/custom-plugins.md

@@ -15,7 +15,7 @@ When naming your plugin, keep in mind that the name needs to conform to these ru
 - have two parts separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same name cannot not already exist
+- a plugin with the same name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.5/custom-plugins.md

@@ -37,7 +37,7 @@ When naming your plugin, keep in mind that the full name needs to conform to the
 - have two parts, prefix + name, separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same prefix + name cannot not already exist
+- a plugin with the same prefix + name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.6/custom-plugins.md

@@ -37,7 +37,7 @@ When naming your plugin, keep in mind that the full name needs to conform to the
 - have two parts, prefix + name, separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same prefix + name cannot not already exist
+- a plugin with the same prefix + name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.7/custom-plugins.md

@@ -37,7 +37,7 @@ When naming your plugin, keep in mind that the full name needs to conform to the
 - have two parts, prefix + name, separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same prefix + name cannot not already exist
+- a plugin with the same prefix + name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.8/custom-plugins.md

@@ -37,7 +37,7 @@ When naming your plugin, keep in mind that the full name needs to conform to the
 - have two parts, prefix + name, separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same prefix + name cannot not already exist
+- a plugin with the same prefix + name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.9/custom-plugins.md

@@ -37,7 +37,7 @@ When naming your plugin, keep in mind that the full name needs to conform to the
 - have two parts, prefix + name, separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same prefix + name cannot not already exist
+- a plugin with the same prefix + name cannot already exist
 
 ### Some examples:
 

test/e2e/backup/backup.go

@@ -31,15 +31,33 @@ import (
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
 
+type BackupRestoreTestConfig struct {
+	useVolumeSnapshots  bool
+	kibishiiPatchSubDir string
+	isRetainPVTest      bool
+}
+
 func BackupRestoreWithSnapshots() {
-	BackupRestoreTest(true)
+	config := BackupRestoreTestConfig{true, "", false}
+	BackupRestoreTest(config)
 }
 
 func BackupRestoreWithRestic() {
-	BackupRestoreTest(false)
+	config := BackupRestoreTestConfig{false, "", false}
+	BackupRestoreTest(config)
 }
 
-func BackupRestoreTest(useVolumeSnapshots bool) {
+func BackupRestoreRetainedPVWithSnapshots() {
+	config := BackupRestoreTestConfig{true, "overlays/sc-reclaim-policy/", true}
+	BackupRestoreTest(config)
+}
+
+func BackupRestoreRetainedPVWithRestic() {
+	config := BackupRestoreTestConfig{false, "overlays/sc-reclaim-policy/", true}
+	BackupRestoreTest(config)
+}
+
+func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 
 	var (
 		backupName, restoreName, kibishiiNamespace string
@@ -48,25 +66,34 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 		veleroCfg                                  VeleroConfig
 	)
 	provideSnapshotVolumesParmInBackup = false
+	useVolumeSnapshots := backupRestoreTestConfig.useVolumeSnapshots
 
 	BeforeEach(func() {
 		veleroCfg = VeleroCfg
+
+		veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir
 		veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 		veleroCfg.UseNodeAgent = !useVolumeSnapshots
 		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
 			Skip("Volume snapshots not supported on kind")
 		}
+
 		var err error
 		flag.Parse()
 		UUIDgen, err = uuid.NewRandom()
 		kibishiiNamespace = "k-" + UUIDgen.String()
 		Expect(err).To(Succeed())
+		DeleteStorageClass(context.Background(), *veleroCfg.ClientToInstallVelero, KibishiiStorageClassName)
 	})
 
 	AfterEach(func() {
 		if !veleroCfg.Debug {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
+				if backupRestoreTestConfig.isRetainPVTest {
+					CleanAllRetainedPV(context.Background(), *veleroCfg.ClientToInstallVelero)
+				}
+				DeleteStorageClass(context.Background(), *veleroCfg.ClientToInstallVelero, KibishiiStorageClassName)
 			})
 			if veleroCfg.InstallVelero {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
@@ -106,6 +133,9 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 		})
 
 		It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
+			if backupRestoreTestConfig.isRetainPVTest {
+				Skip("It's tested by 1st test case")
+			}
 			if veleroCfg.AdditionalBSLProvider == "" {
 				Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
 			}

test/e2e/basic/namespace-mapping.go

@@ -102,7 +102,7 @@ func (n *NamespaceMapping) Verify() error {
 		n.kibishiiData.Levels = len(*n.NSIncluded) + index
 		By(fmt.Sprintf("Verify workload %s after restore ", ns), func() {
 			Expect(KibishiiVerifyAfterRestore(n.Client, ns,
-				n.Ctx, n.kibishiiData)).To(Succeed(), "Fail to verify workload after restore")
+				n.Ctx, n.kibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
 		})
 	}
 	for _, ns := range *n.NSIncluded {

test/e2e/e2e_suite_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
 
+	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	. "github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test/e2e/backup"
 	. "github.com/vmware-tanzu/velero/test/e2e/backups"
@@ -49,6 +50,7 @@ import (
 )
 
 func init() {
+	VeleroCfg.Options = &install.Options{}
 	flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into.  Required.")
 	flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
 	flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
@@ -102,6 +104,10 @@ var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provi
 
 var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
 
+var _ = Describe("[Basic][Snapshot][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithSnapshots)
+
+var _ = Describe("[Basic][Restic][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithRestic)
+
 var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest)
 
 var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSBackupRestore)

test/e2e/migration/migration.go

@@ -273,15 +273,16 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 			}
 
 			By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
+				//Ensure workload of "migrationNamespace" existed in cluster-A
 				ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace)
 				Expect(ns.Name).To(Equal(migrationNamespace))
-				Expect(err).NotTo(HaveOccurred())
+				Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in cluster-B err: %v", err))
 
+				//Ensure cluster-B is the target cluster
 				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
 				_, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace)
 				Expect(err).To(HaveOccurred())
 				strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")
-
 				fmt.Println(err)
 
 				veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
@@ -335,7 +336,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 
 			By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() {
 				Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace,
-					oneHourTimeout, &KibishiiData)).To(Succeed(), "Fail to verify workload after restore")
+					oneHourTimeout, &KibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
 			})
 
 			// TODO: delete backup created by case self, not all

test/e2e/pv-backup/pv-backup-filter.go

@@ -180,7 +180,7 @@ func fileContent(namespace, podName, volume string) string {
 }
 
 func fileExist(ctx context.Context, namespace, podName, volume string) error {
-	c, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
+	c, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s ",
 			FILE_NAME, volume, podName, namespace))
@@ -195,7 +195,7 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error {
 	}
 }
 func fileNotExist(ctx context.Context, namespace, podName, volume string) error {
-	_, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
+	_, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return nil
 	} else {

test/e2e/resourcepolicies/resource_policies.go

@@ -24,7 +24,6 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -164,7 +163,7 @@ func (r *ResourcePoliciesCase) Verify() error {
 					if vol.Name != volName {
 						continue
 					}
-					content, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName)
+					content, _, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName)
 					if i%2 == 0 {
 						Expect(err).To(HaveOccurred(), "Expected file not found") // File should not exist
 					} else {

test/e2e/upgrade/upgrade.go

@@ -29,7 +29,6 @@ import (
 	. "github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/util/kibishii"
-
 	. "github.com/vmware-tanzu/velero/test/util/providers"
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
@@ -256,7 +255,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 
 			By(fmt.Sprintf("Verify workload %s after restore ", upgradeNamespace), func() {
 				Expect(KibishiiVerifyAfterRestore(*veleroCfg.ClientToInstallVelero, upgradeNamespace,
-					oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore")
+					oneHourTimeout, DefaultKibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
 			})
 		})
 	})

test/perf/Makefile

@@ -76,6 +76,17 @@ NFS_SERVER_PATH ?=
 UPLOADER_TYPE ?=
 TEST_CASE_DESCRIBE ?= 'velero performance test'
 BACKUP_FOR_RESTORE ?=
+Delete_Cluster_Resource ?= false
+Debug_Velero_Pod_Restart ?= false
+NODE_AGENT_POD_CPU_LIMIT ?= 4
+NODE_AGENT_POD_MEM_LIMIT ?= 4Gi
+NODE_AGENT_POD_CPU_REQUEST ?= 2
+NODE_AGENT_POD_MEM_REQUEST ?= 2Gi
+VELERO_POD_CPU_LIMIT ?= 4
+VELERO_POD_MEM_LIMIT ?= 4Gi
+VELERO_POD_CPU_REQUEST ?= 2
+VELERO_POD_MEM_REQUEST ?= 2Gi
+POD_VOLUME_OPERATION_TIMEOUT ?= 6h
 
 .PHONY:ginkgo
 ginkgo: # Make sure ginkgo is in $GOPATH/bin
@@ -110,7 +121,18 @@ run: ginkgo
 		-uploader-type=$(UPLOADER_TYPE) \
 		-nfs-server-path=$(NFS_SERVER_PATH) \
 		-test-case-describe=$(TEST_CASE_DESCRIBE) \
-		-backup-for-restore=$(BACKUP_FOR_RESTORE)
+		-backup-for-restore=$(BACKUP_FOR_RESTORE) \
+		-delete-cluster-resource=$(Delete_Cluster_Resource) \
+		-debug-velero-pod-restart=$(Debug_Velero_Pod_Restart) \
+		-node-agent-pod-cpu-limit=$(NODE_AGENT_POD_CPU_LIMIT) \
+		-node-agent-pod-mem-limit=$(NODE_AGENT_POD_MEM_LIMIT) \
+		-node-agent-pod-cpu-request=$(NODE_AGENT_POD_CPU_REQUEST) \
+		-node-agent-pod-mem-request=$(NODE_AGENT_POD_MEM_REQUEST) \
+		-velero-pod-cpu-limit=$(VELERO_POD_CPU_LIMIT) \
+		-velero-pod-mem-limit=$(VELERO_POD_MEM_LIMIT) \
+		-velero-pod-cpu-request=$(VELERO_POD_CPU_REQUEST) \
+		-velero-pod-mem-request=$(VELERO_POD_MEM_REQUEST) \
+		-pod-volume-operation-timeout=$(POD_VOLUME_OPERATION_TIMEOUT)
 
 build: ginkgo
 	mkdir -p $(OUTPUT_DIR)

test/perf/backup/backup.go

@@ -32,7 +32,7 @@ type BackupTest struct {
 
 func (b *BackupTest) Init() error {
 	b.TestCase.Init()
-	b.Ctx, b.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour)
+	b.Ctx, b.CtxCancel = context.WithTimeout(context.Background(), 6*time.Hour)
 	b.CaseBaseName = "backup"
 	b.BackupName = "backup-" + b.CaseBaseName + "-" + b.UUIDgen
 

test/perf/basic/basic.go

@@ -18,12 +18,14 @@ package basic
 
 import (
 	"context"
-	"fmt"
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
+
 	. "github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test/perf/test"
+	"github.com/vmware-tanzu/velero/test/util/k8s"
 )
 
 type BasicTest struct {
@@ -32,7 +34,7 @@ type BasicTest struct {
 
 func (b *BasicTest) Init() error {
 	b.TestCase.Init()
-	b.Ctx, b.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour)
+	b.Ctx, b.CtxCancel = context.WithTimeout(context.Background(), 6*time.Hour)
 	b.CaseBaseName = "backuprestore"
 	b.BackupName = "backup-" + b.CaseBaseName + "-" + b.UUIDgen
 	b.RestoreName = "restore-" + b.CaseBaseName + "-" + b.UUIDgen
@@ -49,10 +51,20 @@ func (b *BasicTest) Init() error {
 		"--from-backup", b.BackupName, "--wait",
 	}
 
+	if !VeleroCfg.DeleteClusterResource {
+		joinedNsMapping, err := k8s.GetMappingNamespaces(b.Ctx, b.Client, *b.NSExcluded)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get mapping namespaces in init")
+		}
+
+		b.RestoreArgs = append(b.RestoreArgs, "--namespace-mappings")
+		b.RestoreArgs = append(b.RestoreArgs, joinedNsMapping)
+	}
+
 	b.TestMsg = &TestMSG{
 		Desc:      "Do backup and restore resources for performance test",
 		FailedMSG: "Failed to backup and restore resources",
-		Text:      fmt.Sprintf("Should backup and restore resources success"),
+		Text:      "Should backup and restore resources success",
 	}
 	return nil
 }

test/perf/e2e_suite_test.go

@@ -21,12 +21,14 @@ import (
 	"flag"
 	"fmt"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
 
+	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	. "github.com/vmware-tanzu/velero/test"
 
 	"github.com/vmware-tanzu/velero/test/perf/backup"
@@ -39,6 +41,7 @@ import (
 )
 
 func init() {
+	VeleroCfg.Options = &install.Options{}
 	flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into.  Required.")
 	flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
 	flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
@@ -56,6 +59,15 @@ func init() {
 	flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test.  Optional.")
 	flag.BoolVar(&VeleroCfg.UseNodeAgent, "use-node-agent", true, "whether deploy node agent daemonset velero during the test.  Optional.")
 	flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
+	flag.StringVar(&VeleroCfg.NodeAgentPodCPULimit, "node-agent-pod-cpu-limit", "4", "CPU limit for node agent pod. Optional.")
+	flag.StringVar(&VeleroCfg.NodeAgentPodMemLimit, "node-agent-pod-mem-limit", "4Gi", "Memory limit for node agent pod. Optional.")
+	flag.StringVar(&VeleroCfg.NodeAgentPodCPURequest, "node-agent-pod-cpu-request", "2", "CPU request for node agent pod. Optional.")
+	flag.StringVar(&VeleroCfg.NodeAgentPodMemRequest, "node-agent-pod-mem-request", "2Gi", "Memory request for node agent pod. Optional.")
+	flag.StringVar(&VeleroCfg.VeleroPodCPULimit, "velero-pod-cpu-limit", "4", "CPU limit for velero pod. Optional.")
+	flag.StringVar(&VeleroCfg.VeleroPodMemLimit, "velero-pod-mem-limit", "4Gi", "Memory limit for velero pod. Optional.")
+	flag.StringVar(&VeleroCfg.VeleroPodCPURequest, "velero-pod-cpu-request", "2", "CPU request for velero pod. Optional.")
+	flag.StringVar(&VeleroCfg.VeleroPodMemRequest, "velero-pod-mem-request", "2Gi", "Memory request for velero pod. Optional.")
+	flag.DurationVar(&VeleroCfg.PodVolumeOperationTimeout, "pod-volume-operation-timeout", 360*time.Minute, "Timeout for pod volume operations. Optional.")
 	//vmware-tanzu-experiments
 	flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.")
 	flag.StringVar(&VeleroCfg.DefaultCluster, "default-cluster-context", "", "Default cluster context for migration test.")
@@ -65,6 +77,8 @@ func init() {
 	flag.StringVar(&VeleroCfg.NFSServerPath, "nfs-server-path", "", "the path of nfs server")
 	flag.StringVar(&VeleroCfg.TestCaseDescribe, "test-case-describe", "velero performance test", "the description for the current test")
 	flag.StringVar(&VeleroCfg.BackupForRestore, "backup-for-restore", "", "the name of backup for restore")
+	flag.BoolVar(&VeleroCfg.DeleteClusterResource, "delete-cluster-resource", false, "delete cluster resource after test")
+	flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "Switch for debugging velero pod restart.")
 }
 
 func initConfig() error {

test/perf/metrics/pod.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -29,6 +30,7 @@ import (
 )
 
 const PodResourceDesc = "Resource consumption"
+const PodMetricsTimeout = 5 * time.Minute
 
 type PodMetrics struct {
 	Client             *metricsclientset.Clientset
@@ -39,31 +41,31 @@ type PodMetrics struct {
 }
 
 func (p *PodMetrics) Update() error {
-	cpu, mem, err := metrics.GetPodUsageMetrics(p.Ctx, p.Client, p.PodName, p.Namespace)
+	cpu, mem, err := metrics.GetPodUsageMetrics(p.Ctx, p.Client, p.PodName, p.Namespace, PodMetricsTimeout)
 	if err != nil {
 		return errors.WithStack(err)
-	} else {
-		keyMaxCPU := p.PodName + ":MaxCPU"
-		curCPU := cpu.MilliValue()
-		if curCPU > p.Metrics[keyMaxCPU] {
-			p.Metrics[keyMaxCPU] = curCPU
-		}
-
-		keyMaxMem := p.PodName + ":MaxMemory"
-		curMem := mem.MilliValue()
-		if curMem > p.Metrics[keyMaxMem] {
-			p.Metrics[keyMaxMem] = curMem
-		}
-
-		keyAvgCPU := p.PodName + ":AverageCPU"
-		preAvgCPU := p.Metrics[keyAvgCPU]
-		p.Metrics[keyAvgCPU] = (preAvgCPU*p.count + curCPU) / (p.count + 1)
-
-		keyAvgMem := p.PodName + ":AverageMemory"
-		preAvgMem := p.Metrics[keyAvgMem]
-		p.Metrics[keyAvgMem] = (preAvgMem*p.count + curMem) / (p.count + 1)
-		p.count++
 	}
+	keyMaxCPU := p.PodName + ":MaxCPU"
+	curCPU := cpu.MilliValue()
+	if curCPU > p.Metrics[keyMaxCPU] {
+		p.Metrics[keyMaxCPU] = curCPU
+	}
+
+	keyMaxMem := p.PodName + ":MaxMemory"
+	curMem := mem.MilliValue()
+	if curMem > p.Metrics[keyMaxMem] {
+		p.Metrics[keyMaxMem] = curMem
+	}
+
+	keyAvgCPU := p.PodName + ":AverageCPU"
+	preAvgCPU := p.Metrics[keyAvgCPU]
+	p.Metrics[keyAvgCPU] = (preAvgCPU*p.count + curCPU) / (p.count + 1)
+
+	keyAvgMem := p.PodName + ":AverageMemory"
+	preAvgMem := p.Metrics[keyAvgMem]
+	p.Metrics[keyAvgMem] = (preAvgMem*p.count + curMem) / (p.count + 1)
+	p.count++
+
 	return nil
 }
 

test/perf/metrics/time.go

@@ -16,40 +16,53 @@ limitations under the License.
 
 package metrics
 
-import "time"
+import (
+	"fmt"
+	"time"
+)
 
 const TimeCaseDesc = "Time cost"
 
+type TimeSpan struct {
+	Start time.Time
+	End   time.Time
+}
+
 type TimeMetrics struct {
 	Name     string
-	TimeInfo map[string]time.Time // metric name : start timestamp
-	Metrics  map[string]float64   // metric name : time duration
+	TimeInfo map[string]TimeSpan // metric name : start timestamp
 }
 
 func (t *TimeMetrics) GetMetrics() map[string]string {
 	tmpMetrics := make(map[string]string)
-	for k, v := range t.Metrics {
-		duration := time.Duration(v) * time.Second
-		tmpMetrics[k] = duration.String()
+	for k, v := range t.TimeInfo {
+		duration := v.End.Sub(v.Start)
+		if duration < time.Second {
+			// For those too shoter time difference we should ignored
+			// as it may not really execute the logic
+			continue
+		}
+		tmpMetrics[k] = duration.String() + fmt.Sprintf(" (%s - %s)", v.Start.Format(time.RFC3339), v.End.Format(time.RFC3339))
 	}
 	return tmpMetrics
 }
 
 func (t *TimeMetrics) Start(name string) {
-	t.TimeInfo[name] = time.Now()
-}
-
-func (t *TimeMetrics) End(name string) {
-	t.Metrics[name] = time.Now().Sub(t.TimeInfo[name]).Seconds()
-	if t.Metrics[name] < 1 {
-		// For those too shoter time difference we should ignored
-		// as it may not really execute the logic
-		delete(t.Metrics, name)
+	t.TimeInfo[name] = TimeSpan{
+		Start: time.Now(),
 	}
 }
 
+func (t *TimeMetrics) End(name string) {
+	if _, ok := t.TimeInfo[name]; !ok {
+		return
+	}
+	timeSpan := t.TimeInfo[name]
+	timeSpan.End = time.Now()
+	t.TimeInfo[name] = timeSpan
+}
+
 func (t *TimeMetrics) Update() error {
-	t.Metrics[t.Name] = time.Now().Sub(t.TimeInfo[t.Name]).Seconds()
 	return nil
 }
 

test/perf/restore/restore.go

@@ -25,6 +25,7 @@ import (
 
 	. "github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test/perf/test"
+	"github.com/vmware-tanzu/velero/test/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
 
@@ -34,7 +35,7 @@ type RestoreTest struct {
 
 func (r *RestoreTest) Init() error {
 	r.TestCase.Init()
-	r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour)
+	r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 6*time.Hour)
 	r.CaseBaseName = "restore"
 	r.RestoreName = "restore-" + r.CaseBaseName + "-" + r.UUIDgen
 
@@ -43,7 +44,7 @@ func (r *RestoreTest) Init() error {
 		FailedMSG: "Failed to restore resources",
 		Text:      fmt.Sprintf("Should restore resources success"),
 	}
-	return r.clearUpResourcesBeforRestore()
+	return nil
 }
 
 func (r *RestoreTest) clearUpResourcesBeforRestore() error {
@@ -52,6 +53,11 @@ func (r *RestoreTest) clearUpResourcesBeforRestore() error {
 }
 
 func (r *RestoreTest) Restore() error {
+	// we need to clear up all resources before do the restore test
+	err := r.clearUpResourcesBeforRestore()
+	if err != nil {
+		return errors.Wrapf(err, "failed to clear up resources before do the restore test")
+	}
 	var backupName string
 	if VeleroCfg.BackupForRestore != "" {
 		backupName = VeleroCfg.BackupForRestore
@@ -71,6 +77,16 @@ func (r *RestoreTest) Restore() error {
 		"--from-backup", r.BackupName, "--wait",
 	}
 
+	if !VeleroCfg.DeleteClusterResource {
+		joinedNsMapping, err := k8s.GetMappingNamespaces(r.Ctx, r.Client, *r.NSExcluded)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get mapping namespaces in init")
+		}
+
+		r.RestoreArgs = append(r.RestoreArgs, "--namespace-mappings")
+		r.RestoreArgs = append(r.RestoreArgs, joinedNsMapping)
+	}
+
 	return r.TestCase.Restore()
 }
 func (r *RestoreTest) Destroy() error {

test/perf/test/test.go

@@ -97,14 +97,15 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
 }
 
 func (t *TestCase) Init() error {
-	t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour)
+	t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 6*time.Hour)
 	t.NSExcluded = &[]string{"kube-system", "velero", "default", "kube-public", "kube-node-lease"}
 	t.UUIDgen = t.GenerateUUID()
 	t.Client = *VeleroCfg.DefaultClient
 	t.timer = &metrics.TimeMetrics{
-		Name:     "Total time cost",
-		TimeInfo: map[string]time.Time{"Total time cost": time.Now()},
-		Metrics:  make(map[string]float64),
+		Name: "Total time cost",
+		TimeInfo: map[string]metrics.TimeSpan{"Total time cost": {
+			Start: time.Now(),
+		}},
 	}
 	return nil
 }
@@ -131,10 +132,12 @@ func (t *TestCase) Backup() error {
 }
 
 func (t *TestCase) Destroy() error {
-	By(fmt.Sprintf("Start to destroy namespace %s......", t.CaseBaseName), func() {
-		Expect(CleanupNamespacesFiterdByExcludes(t.GetTestCase().Ctx, t.Client, *t.NSExcluded)).To(Succeed(), "Could cleanup retrieve namespaces")
-		Expect(ClearClaimRefForFailedPVs(t.Ctx, t.Client)).To(Succeed(), "Failed to make PV status become to available")
-	})
+	if VeleroCfg.DeleteClusterResource {
+		By(fmt.Sprintf("Start to destroy namespace %s......", t.CaseBaseName), func() {
+			Expect(CleanupNamespacesFiterdByExcludes(t.GetTestCase().Ctx, t.Client, *t.NSExcluded)).To(Succeed(), "Could cleanup retrieve namespaces")
+			Expect(ClearClaimRefForFailedPVs(t.Ctx, t.Client)).To(Succeed(), "Failed to make PV status become to available")
+		})
+	}
 	return nil
 }
 
@@ -160,7 +163,7 @@ func (t *TestCase) Verify() error {
 }
 
 func (t *TestCase) Clean() error {
-	if !VeleroCfg.Debug {
+	if !VeleroCfg.Debug || VeleroCfg.DeleteClusterResource {
 		By("Clean backups and restore after test", func() {
 			if len(t.BackupArgs) != 0 {
 				if err := VeleroBackupDelete(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName); err != nil {
@@ -269,8 +272,7 @@ func (t *TestCase) MonitorMetircs(ctx context.Context, collectors *metrics.Metri
 
 	timeMetrics := &metrics.TimeMetrics{
 		Name:     t.CaseBaseName,
-		TimeInfo: make(map[string]time.Time),
-		Metrics:  make(map[string]float64),
+		TimeInfo: make(map[string]metrics.TimeSpan),
 	}
 	collectors.RegisterOneTimeMetric(timeMetrics)
 

test/types.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/google/uuid"
 
+	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 )
 
@@ -40,6 +41,7 @@ var ReportData *Report
 
 type VeleroConfig struct {
 	VeleroCfgInPerf
+	*install.Options
 	VeleroCLI                         string
 	VeleroImage                       string
 	VeleroVersion                     string
@@ -66,7 +68,6 @@ type VeleroConfig struct {
 	AddBSLPlugins                     string
 	InstallVelero                     bool
 	KibishiiDirectory                 string
-	Features                          string
 	Debug                             bool
 	GCFrequency                       string
 	DefaultCluster                    string
@@ -74,12 +75,7 @@ type VeleroConfig struct {
 	ClientToInstallVelero             *TestClient
 	DefaultClient                     *TestClient
 	StandbyClient                     *TestClient
-	UploaderType                      string
-	UseNodeAgent                      bool
-	UseRestic                         bool
 	ProvideSnapshotsVolumeParam       bool
-	DefaultVolumesToFsBackup          bool
-	UseVolumeSnapshots                bool
 	VeleroServerDebugMode             bool
 	SnapshotMoveData                  bool
 	DataMoverPlugin                   string
@@ -90,9 +86,10 @@ type VeleroConfig struct {
 }
 
 type VeleroCfgInPerf struct {
-	NFSServerPath    string
-	TestCaseDescribe string
-	BackupForRestore string
+	NFSServerPath         string
+	TestCaseDescribe      string
+	BackupForRestore      string
+	DeleteClusterResource bool
 }
 
 type SnapshotCheckPoint struct {

test/util/csi/common.go

@@ -21,14 +21,12 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 )
 
@@ -128,6 +126,7 @@ func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, err
 	}
 	return snapshotHandleList, nil
 }
+
 func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
 	pvcList, err := GetPvcByPVCName(context.Background(), namespace, podName)
 	if err != nil {

test/util/k8s/common.go

@@ -104,7 +104,6 @@ func GetPvcByPVCName(ctx context.Context, namespace, pvcName string) ([]string,
 		Args: []string{"{print $1}"},
 	}
 	cmds = append(cmds, cmd)
-
 	return common.GetListByCmdPipes(ctx, cmds)
 }
 
@@ -279,15 +278,30 @@ func CreateFileToPod(ctx context.Context, namespace, podName, containerName, vol
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
 	return cmd.Run()
 }
-func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, error) {
+func FileExistInPV(ctx context.Context, namespace, podName, containerName, volume, filename string) (bool, error) {
+	stdout, stderr, err := ReadFileFromPodVolume(ctx, namespace, podName, containerName, volume, filename)
+
+	output := fmt.Sprintf("%s:%s", stdout, stderr)
+	if strings.Contains(output, fmt.Sprintf("/%s/%s: No such file or directory", volume, filename)) {
+		return false, nil
+	} else {
+		if err == nil {
+			return true, nil
+		} else {
+			return false, errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s",
+				filename, volume, podName, namespace))
+		}
+	}
+}
+func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, string, error) {
 	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
 		"--", "cat", fmt.Sprintf("/%s/%s", volume, filename)}
 	cmd := exec.CommandContext(ctx, "kubectl", arg...)
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
 	stdout, stderr, err := veleroexec.RunCommand(cmd)
-	fmt.Print(stdout)
-	fmt.Print(stderr)
-	return stdout, err
+	fmt.Printf("stdout: %s\n", stdout)
+	fmt.Printf("stderr: %s\n", stderr)
+	return stdout, stderr, err
 }
 
 func RunCommand(cmdName string, arg []string) string {

test/util/k8s/namespace.go

@@ -194,3 +194,42 @@ func NamespaceShouldNotExist(ctx context.Context, client TestClient, namespace s
 	}
 	return nil
 }
+
+func GetBackupNamespaces(ctx context.Context, client TestClient, excludeNS []string) ([]string, error) {
+	namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not retrieve namespaces")
+	}
+	var backupNamespaces []string
+	for _, checkNamespace := range namespaces.Items {
+		isExclude := false
+		for k := range excludeNS {
+			if checkNamespace.Name == excludeNS[k] {
+				isExclude = true
+			}
+		}
+		if !isExclude {
+			backupNamespaces = append(backupNamespaces, checkNamespace.Name)
+		}
+	}
+	return backupNamespaces, nil
+}
+
+func GetMappingNamespaces(ctx context.Context, client TestClient, excludeNS []string) (string, error) {
+	ns, err := GetBackupNamespaces(ctx, client, excludeNS)
+	if err != nil {
+		return "", errors.Wrap(err, "Could not retrieve namespaces")
+	} else if len(ns) == 0 {
+		return "", errors.Wrap(err, "Get empty namespaces in backup")
+	}
+
+	nsMapping := []string{}
+	for _, n := range ns {
+		nsMapping = append(nsMapping, n+":mapping-"+n)
+	}
+	joinedNsMapping := strings.Join(nsMapping, ",")
+	if len(joinedNsMapping) > 0 {
+		joinedNsMapping = joinedNsMapping[:len(joinedNsMapping)-1]
+	}
+	return joinedNsMapping, nil
+}

test/util/k8s/persistentvolumes.go

@@ -22,10 +22,9 @@ import (
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/client-go/util/retry"
-
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
 )
 
 func CreatePersistentVolume(client TestClient, name string) (*corev1.PersistentVolume, error) {
@@ -93,3 +92,16 @@ func ClearClaimRefForFailedPVs(ctx context.Context, client TestClient) error {
 
 	return nil
 }
+
+func GetAllPVNames(ctx context.Context, client TestClient) ([]string, error) {
+	var pvNameList []string
+	pvList, err := client.ClientGo.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to List PV")
+	}
+
+	for _, pvName := range pvList.Items {
+		pvNameList = append(pvNameList, pvName.Name)
+	}
+	return pvNameList, nil
+}

test/util/kibishii/kibishii_utils.go

@@ -51,6 +51,7 @@ type KibishiiData struct {
 var DefaultKibishiiWorkerCounts = 2
 var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, DefaultKibishiiWorkerCounts}
 
+var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"}
 var KibishiiPVCNameList = []string{"kibishii-data-kibishii-deployment-0", "kibishii-data-kibishii-deployment-1"}
 var KibishiiStorageClassName = "kibishii-storage-class"
 
@@ -107,6 +108,8 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 	}
 
 	fmt.Printf("VeleroBackupNamespace done %s\n", time.Now().Format("2006-01-02 15:04:05"))
+
+	// Checkpoint for a successful backup
 	if useVolumeSnapshots {
 		if providerName == "vsphere" {
 			// Wait for uploads started by the Velero Plugin for vSphere to complete
@@ -165,11 +168,49 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 		}
 	}
 
+	// Modify PV data right after backup. If PV's reclaim policy is retain, PV will be restored with the origin resource config
+	fileName := "file-" + kibishiiNamespace
+	fileBaseContent := fileName
+	fmt.Printf("Re-poulate volume  %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	for _, pod := range KibishiiPodNameList {
+		// To ensure Kibishii verification result is accurate
+		ClearKibishiiData(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data")
+
+		fileContent := fileBaseContent + pod
+		err := CreateFileToPod(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data",
+			fileName, fileContent)
+		if err != nil {
+			return errors.Wrapf(err, "failed to create file %s", fileName)
+		}
+	}
+	fmt.Printf("Re-poulate volume done %s\n", time.Now().Format("2006-01-02 15:04:05"))
+
+	pvList := []string{}
+	if strings.Contains(veleroCfg.KibishiiDirectory, "sc-reclaim-policy") {
+		// Get leftover PV list for PV cleanup
+		for _, pvc := range KibishiiPVCNameList {
+			pv, err := GetPvName(oneHourTimeout, client, pvc, kibishiiNamespace)
+			if err != nil {
+				errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
+			}
+			pvList = append(pvList, pv)
+		}
+	}
+
 	fmt.Printf("Simulating a disaster by removing namespace %s %s\n", kibishiiNamespace, time.Now().Format("2006-01-02 15:04:05"))
 	if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
 		return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
 	}
 
+	if strings.Contains(veleroCfg.KibishiiDirectory, "sc-reclaim-policy") {
+		// In scenario of CSI PV-retain-policy test, to restore PV of the backed up resource, we should make sure
+		// there are no PVs of the same name left, because in previous test step, PV's reclaim policy is retain,
+		// so PVs are not deleted although workload namespace is destroyed.
+		if err := DeletePVs(oneHourTimeout, *veleroCfg.ClientToInstallVelero, pvList); err != nil {
+			return errors.Wrapf(err, "failed to delete PVs %v", pvList)
+		}
+	}
+
 	// the snapshots of AWS may be still in pending status when do the restore, wait for a while
 	// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
 	// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
@@ -191,10 +232,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 			return errors.New(fmt.Sprintf("PVR count %d is not as expected %d", len(pvrs), pvCount))
 		}
 	}
+
 	fmt.Printf("KibishiiVerifyAfterRestore %s\n", time.Now().Format("2006-01-02 15:04:05"))
-	if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData); err != nil {
+	if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData, fileName); err != nil {
 		return errors.Wrapf(err, "Error verifying kibishii after restore")
 	}
+
 	fmt.Printf("kibishii test completed successfully %s\n", time.Now().Format("2006-01-02 15:04:05"))
 	return nil
 }
@@ -309,6 +352,15 @@ func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespa
 	return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
 }
 
+func KibishiiGenerateData(oneHourTimeout context.Context, kibishiiNamespace string, kibishiiData *KibishiiData) error {
+	fmt.Printf("generateData %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	if err := generateData(oneHourTimeout, kibishiiNamespace, kibishiiData); err != nil {
+		return errors.Wrap(err, "Failed to generate data")
+	}
+	fmt.Printf("generateData done %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	return nil
+}
+
 func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient,
 	providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures,
 	kibishiiDirectory string, useVolumeSnapshots bool, kibishiiData *KibishiiData) error {
@@ -338,16 +390,12 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie
 	if kibishiiData == nil {
 		kibishiiData = DefaultKibishiiData
 	}
-	fmt.Printf("generateData %s\n", time.Now().Format("2006-01-02 15:04:05"))
-	if err := generateData(oneHourTimeout, kibishiiNamespace, kibishiiData); err != nil {
-		return errors.Wrap(err, "Failed to generate data")
-	}
-	fmt.Printf("generateData done %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	KibishiiGenerateData(oneHourTimeout, kibishiiNamespace, kibishiiData)
 	return nil
 }
 
 func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, oneHourTimeout context.Context,
-	kibishiiData *KibishiiData) error {
+	kibishiiData *KibishiiData, incrementalFileName string) error {
 	if kibishiiData == nil {
 		kibishiiData = DefaultKibishiiData
 	}
@@ -357,6 +405,18 @@ func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, one
 	if err := waitForKibishiiPods(oneHourTimeout, client, kibishiiNamespace); err != nil {
 		return errors.Wrapf(err, "Failed to wait for ready status of kibishii pods in %s", kibishiiNamespace)
 	}
+	if incrementalFileName != "" {
+		for _, pod := range KibishiiPodNameList {
+			exist, err := FileExistInPV(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data", incrementalFileName)
+			if err != nil {
+				return errors.Wrapf(err, fmt.Sprintf("fail to get file %s", incrementalFileName))
+			}
+
+			if exist {
+				return errors.New("Unexpected incremental data exist")
+			}
+		}
+	}
 
 	// TODO - check that namespace exists
 	fmt.Printf("running kibishii verify\n")
@@ -365,3 +425,11 @@ func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, one
 	}
 	return nil
 }
+
+func ClearKibishiiData(ctx context.Context, namespace, podName, containerName, dir string) error {
+	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
+		"--", "/bin/sh", "-c", "rm -rf /" + dir + "/*"}
+	cmd := exec.CommandContext(ctx, "kubectl", arg...)
+	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
+	return cmd.Run()
+}

test/util/metrics/pod.go

@@ -18,21 +18,35 @@ package metrics
 
 import (
 	"context"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/metrics/pkg/apis/metrics/v1beta1"
 	metricsclientset "k8s.io/metrics/pkg/client/clientset/versioned"
 )
 
-func GetPodUsageMetrics(ctx context.Context, metricsClient *metricsclientset.Clientset, podName, namespace string) (cpuUsage, memoryUsage resource.Quantity, err error) {
+func GetPodUsageMetrics(ctx context.Context, metricsClient *metricsclientset.Clientset, podName, namespace string, podMetricsTimeout time.Duration) (cpuUsage, memoryUsage resource.Quantity, err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), podMetricsTimeout)
+	defer cancel()
+
 	var podMetrics *v1beta1.PodMetrics
-	podMetrics, err = metricsClient.MetricsV1beta1().PodMetricses(namespace).Get(ctx, podName, metav1.GetOptions{})
+	err = wait.PollImmediateUntil(time.Second, func() (bool, error) {
+		var err error
+		podMetrics, err = metricsClient.MetricsV1beta1().PodMetricses(namespace).Get(ctx, podName, metav1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	}, ctx.Done())
+
 	if err != nil {
 		return
+	} else if podMetrics == nil {
+		return cpuUsage, memoryUsage, nil
 	}
-
 	// Variables to store the max and sum of CPU and memory usage
 	// For velero pod we only return the main container
 	for _, container := range podMetrics.Containers {

test/util/velero/install.go

@@ -120,6 +120,15 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 	veleroInstallOptions.UploaderType = veleroCfg.UploaderType
 	GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency)
 	veleroInstallOptions.GarbageCollectionFrequency = GCFrequency
+	veleroInstallOptions.PodVolumeOperationTimeout = veleroCfg.PodVolumeOperationTimeout
+	veleroInstallOptions.NodeAgentPodCPULimit = veleroCfg.NodeAgentPodCPULimit
+	veleroInstallOptions.NodeAgentPodCPURequest = veleroCfg.NodeAgentPodCPURequest
+	veleroInstallOptions.NodeAgentPodMemLimit = veleroCfg.NodeAgentPodMemLimit
+	veleroInstallOptions.NodeAgentPodMemRequest = veleroCfg.NodeAgentPodMemRequest
+	veleroInstallOptions.VeleroPodCPULimit = veleroCfg.VeleroPodCPULimit
+	veleroInstallOptions.VeleroPodCPURequest = veleroCfg.VeleroPodCPURequest
+	veleroInstallOptions.VeleroPodMemLimit = veleroCfg.VeleroPodMemLimit
+	veleroInstallOptions.VeleroPodMemRequest = veleroCfg.VeleroPodMemRequest
 
 	err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{
 		Options:                veleroInstallOptions,
@@ -251,6 +260,42 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 		args = append(args, fmt.Sprintf("--garbage-collection-frequency=%v", options.GarbageCollectionFrequency))
 	}
 
+	if options.PodVolumeOperationTimeout > 0 {
+		args = append(args, fmt.Sprintf("--pod-volume-operation-timeout=%v", options.PodVolumeOperationTimeout))
+	}
+
+	if options.NodeAgentPodCPULimit != "" {
+		args = append(args, fmt.Sprintf("--node-agent-pod-cpu-limit=%v", options.NodeAgentPodCPULimit))
+	}
+
+	if options.NodeAgentPodCPURequest != "" {
+		args = append(args, fmt.Sprintf("--node-agent-pod-cpu-request=%v", options.NodeAgentPodCPURequest))
+	}
+
+	if options.NodeAgentPodMemLimit != "" {
+		args = append(args, fmt.Sprintf("--node-agent-pod-mem-limit=%v", options.NodeAgentPodMemLimit))
+	}
+
+	if options.NodeAgentPodMemRequest != "" {
+		args = append(args, fmt.Sprintf("--node-agent-pod-mem-request=%v", options.NodeAgentPodMemRequest))
+	}
+
+	if options.VeleroPodCPULimit != "" {
+		args = append(args, fmt.Sprintf("--velero-pod-cpu-limit=%v", options.VeleroPodCPULimit))
+	}
+
+	if options.VeleroPodCPURequest != "" {
+		args = append(args, fmt.Sprintf("--velero-pod-cpu-request=%v", options.VeleroPodCPURequest))
+	}
+
+	if options.VeleroPodMemLimit != "" {
+		args = append(args, fmt.Sprintf("--velero-pod-mem-limit=%v", options.VeleroPodMemLimit))
+	}
+
+	if options.VeleroPodMemRequest != "" {
+		args = append(args, fmt.Sprintf("--velero-pod-mem-request=%v", options.VeleroPodMemRequest))
+	}
+
 	if len(options.UploaderType) > 0 {
 		args = append(args, fmt.Sprintf("--uploader-type=%v", options.UploaderType))
 	}

test/util/velero/velero_utils.go

@@ -1561,3 +1561,62 @@ func InstallTestStorageClasses(path string) error {
 	}
 	return InstallStorageClass(ctx, tmpFile.Name())
 }
+
+func GetPvName(ctx context.Context, client TestClient, pvcName, namespace string) (string, error) {
+
+	pvcList, err := GetPvcByPVCName(context.Background(), namespace, pvcName)
+	if err != nil {
+		return "", err
+	}
+
+	if len(pvcList) != 1 {
+		return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], pvcName, namespace))
+	}
+
+	pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0])
+	if err != nil {
+		return "", err
+	}
+	if len(pvList) != 1 {
+		return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], pvcName, namespace))
+	}
+
+	return pvList[0], nil
+
+}
+func DeletePVs(ctx context.Context, client TestClient, pvList []string) error {
+	for _, pv := range pvList {
+		args := []string{"delete", "pv", pv, "--timeout=0s"}
+		fmt.Println(args)
+		err := exec.CommandContext(ctx, "kubectl", args...).Run()
+		if err != nil {
+			return errors.New(fmt.Sprintf("Deleted PV  %s ", pv))
+		}
+	}
+	return nil
+}
+
+func CleanAllRetainedPV(ctx context.Context, client TestClient) {
+
+	pvNameList, err := GetAllPVNames(ctx, client)
+	if err != nil {
+		fmt.Println("fail to list PV")
+	}
+	for _, pv := range pvNameList {
+		args := []string{"patch", "pv", pv, "-p", "{\"spec\":{\"persistentVolumeReclaimPolicy\":\"Delete\"}}"}
+		fmt.Println(args)
+		cmd := exec.CommandContext(ctx, "kubectl", args...)
+		stdout, errMsg, err := veleroexec.RunCommand(cmd)
+		if err != nil {
+			fmt.Printf("fail to patch PV %s reclaim policy to delete: stdout: %s, stderr: %s", pv, stdout, errMsg)
+		}
+
+		args = []string{"delete", "pv", pv, "--timeout=60s"}
+		fmt.Println(args)
+		cmd = exec.CommandContext(ctx, "kubectl", args...)
+		stdout, errMsg, err = veleroexec.RunCommand(cmd)
+		if err != nil {
+			fmt.Printf("fail to delete PV %s reclaim policy to delete: stdout: %s, stderr: %s", pv, stdout, errMsg)
+		}
+	}
+}