Merge branch 'main' into issue-fix-6695

.github/workflows/stale-issues.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v3
+      - uses: actions/stale@v6.0.1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           stale-issue-message: "This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 14 days. If a Velero team member has requested log or more information, please provide the output of the shared commands."

changelogs/unreleased/7000-qiuming-best

@@ -0,0 +1 @@
+Make Kopia file parallelism configurable

changelogs/unreleased/7100-blackpiglet

@@ -0,0 +1 @@
+Generate VolumeInfo for backup.

changelogs/unreleased/7115-reasonerjt

@@ -0,0 +1 @@
+Include plugin name in the error message by operations

config/crd/v1/bases/velero.io_backups.yaml

@@ -477,6 +477,15 @@ spec:
                 description: TTL is a time.Duration-parseable string describing how
                   long the Backup should be retained for.
                 type: string
+              uploaderConfig:
+                description: UploaderConfig specifies the configuration for the uploader.
+                nullable: true
+                properties:
+                  parallelFilesUpload:
+                    description: ParallelFilesUpload is the number of files parallel
+                      uploads to perform when using the uploader.
+                    type: integer
+                type: object
               volumeSnapshotLocations:
                 description: VolumeSnapshotLocations is a list containing names of
                   VolumeSnapshotLocations associated with this backup.

config/crd/v1/bases/velero.io_podvolumebackups.yaml

@@ -121,6 +121,14 @@ spec:
                 description: Tags are a map of key-value pairs that should be applied
                   to the volume backup as tags.
                 type: object
+              uploaderConfig:
+                description: UploaderConfig specifies the configuration for the uploader.
+                properties:
+                  parallelFilesUpload:
+                    description: ParallelFilesUpload is the number of files parallel
+                      uploads to perform when using the uploader.
+                    type: integer
+                type: object
               uploaderType:
                 description: UploaderType is the type of the uploader to handle the
                   data transfer.

config/crd/v1/bases/velero.io_schedules.yaml

@@ -514,6 +514,16 @@ spec:
                     description: TTL is a time.Duration-parseable string describing
                       how long the Backup should be retained for.
                     type: string
+                  uploaderConfig:
+                    description: UploaderConfig specifies the configuration for the
+                      uploader.
+                    nullable: true
+                    properties:
+                      parallelFilesUpload:
+                        description: ParallelFilesUpload is the number of files parallel
+                          uploads to perform when using the uploader.
+                        type: integer
+                    type: object
                   volumeSnapshotLocations:
                     description: VolumeSnapshotLocations is a list containing names
                       of VolumeSnapshotLocations associated with this backup.

config/crd/v2alpha1/bases/velero.io_datauploads.yaml

@@ -125,6 +125,14 @@ spec:
                 description: SourcePVC is the name of the PVC which the snapshot is
                   taken for.
                 type: string
+              uploaderConfig:
+                description: UploaderConfig specifies the configuration for the uploader.
+                properties:
+                  parallelFilesUpload:
+                    description: ParallelFilesUpload is the number of files parallel
+                      uploads to perform when using the uploader.
+                    type: integer
+                type: object
             required:
             - backupStorageLocation
             - operationTimeout

design/Implemented/restore-with-EnableAPIGroupVersions-feature.md

@@ -29,7 +29,7 @@ During restore, the proposal is that Velero will determine if the `APIGroupVersi
 The proposed code starts with creating three lists for each backed up resource. The three lists will be created by
   (1) reading the directory names in the backup tarball file and seeing which API group versions were backed up from the source cluster,
   (2) looking at the target cluster and determining which API group versions are supported, and
-  (3) getting config maps from the target cluster in order to get user-defined prioritization of versions.
+  (3) getting ConfigMaps from the target cluster in order to get user-defined prioritization of versions.
 
   The three lists will be used to create a map of chosen versions for each resource to restore. If there is a user-defined list of priority versions, the versions will be checked against the supported versions lists. The highest user-defined priority version that is/was supported by both target and source clusters will be the chosen version for that resource. If no user specified versions are supported by neither target nor source, the versions will be logged and the restore will continue with other prioritizations.
 

design/pv_backup_info.md

@@ -61,7 +61,7 @@ type VolumeInfo struct {
 // CSISnapshotInfo is used for displaying the CSI snapshot status
 type CSISnapshotInfo struct {
     SnapshotHandle  string       // It's the storage provider's snapshot ID for CSI.
-    Size            int64        // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot.
+    Size            int64        // The snapshot corresponding volume size.
 
     Driver          string  // The name of the CSI driver.
     VSCName         string // The name of the VolumeSnapshotContent. 
@@ -70,7 +70,7 @@ type CSISnapshotInfo struct {
 // SnapshotDataMovementInfo is used for displaying the snapshot data mover status.
 type SnapshotDataMovementInfo struct {
     DataMover        string    // The data mover used by the backup. The valid values are `velero` and ``(equals to `velero`).
-    UploaderType     string    // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover.
+    UploaderType     string    // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`.
     RetainedSnapshot string    // The name or ID of the snapshot associated object(SAO). SAO is used to support local snapshots for the snapshot data mover, e.g. it could be a VolumeSnapshot for CSI snapshot data moign/pv_backup_info.
     SnapshotHandle string  	   // It's the filesystem repository's snapshot ID.
 	
@@ -79,7 +79,6 @@ type SnapshotDataMovementInfo struct {
 // VeleroNativeSnapshotInfo is used for displaying the Velero native snapshot status.
 type VeleroNativeSnapshotInfo struct {
     SnapshotHandle      string       // It's the storage provider's snapshot ID for the Velero-native snapshot.
-    Size                int64        // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot.
 
     VolumeType string    // The cloud provider snapshot volume type.
     VolumeAZ   string    // The cloud provider snapshot volume's availability zones.
@@ -89,11 +88,12 @@ type VeleroNativeSnapshotInfo struct {
 // PodVolumeBackupInfo is used for displaying the PodVolumeBackup snapshot status.
 type PodVolumeBackupInfo struct {
     SnapshotHandle      string       // It's the file-system uploader's snapshot ID for PodVolumeBackup.
-    Size                int64        // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot.
+    Size                int64        // The snapshot corresponding volume size.
 
-    UploaderType  string    // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover.
+    UploaderType  string    // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`.
     VolumeName    string   // The PVC's corresponding volume name used by Pod: https://github.com/kubernetes/kubernetes/blob/e4b74dd12fa8cb63c174091d5536a10b8ec19d34/pkg/apis/core/types.go#L48
-    PodName       string   // The Pod name mounting this PVC. The format should be <namespace-name>/<pod-name>.
+    PodName       string   // The Pod name mounting this PVC.
+    PodNamespace  string   // The Pod namespace.
     NodeName      string   // The PVB-taken k8s node's name.
 }
 

design/volume-snapshot-data-movement/volume-snapshot-data-movement.md

@@ -691,7 +691,8 @@ type Provider interface {
 		tags map[string]string,
 		forceFull bool,
 		parentSnapshot string,
-		volMode uploader.PersistentVolumeMode,
+                volMode uploader.PersistentVolumeMode,
+		uploaderCfg shared.UploaderConfig,
 		updater uploader.ProgressUpdater) (string, bool, error)
 
   RunRestore(

internal/resourcemodifiers/json_patch.go

@@ -19,30 +19,37 @@ type JSONPatch struct {
 }
 
 func (p *JSONPatch) ToString() string {
-	if addQuotes(p.Value) {
+	if addQuotes(&p.Value) {
 		return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": "%s"}`, p.Operation, p.From, p.Path, p.Value)
 	}
 	return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": %s}`, p.Operation, p.From, p.Path, p.Value)
 }
 
-func addQuotes(value string) bool {
-	if value == "" {
+func addQuotes(value *string) bool {
+	if *value == "" {
+		return true
+	}
+	// if value is escaped, remove escape and add quotes
+	// this is useful for scenarios where boolean, null and numbers are required to be set as string.
+	if strings.HasPrefix(*value, "\"") && strings.HasSuffix(*value, "\"") {
+		*value = strings.TrimPrefix(*value, "\"")
+		*value = strings.TrimSuffix(*value, "\"")
 		return true
 	}
 	// if value is null, then don't add quotes
-	if value == "null" {
+	if *value == "null" {
 		return false
 	}
 	// if value is a boolean, then don't add quotes
-	if _, err := strconv.ParseBool(value); err == nil {
+	if strings.ToLower(*value) == "true" || strings.ToLower(*value) == "false" {
 		return false
 	}
 	// if value is a json object or array, then don't add quotes.
-	if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
+	if strings.HasPrefix(*value, "{") || strings.HasPrefix(*value, "[") {
 		return false
 	}
 	// if value is a number, then don't add quotes
-	if _, err := strconv.ParseFloat(value, 64); err == nil {
+	if _, err := strconv.ParseFloat(*value, 64); err == nil {
 		return false
 	}
 	return true

internal/resourcemodifiers/resource_modifiers_test.go

@@ -256,6 +256,64 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 			},
 		},
 	}
+	cm9 := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-configmap",
+			Namespace: "test-namespace",
+		},
+		Data: map[string]string{
+			"sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n    groupResource: deployments.apps\n    resourceNameRegex: \"^test-.*$\"\n    namespaces:\n    - bar\n    - foo\n  patches:\n  - operation: replace\n    path: \"/value/bool\"\n    value: \"\\\"true\\\"\"\n\n\n",
+		},
+	}
+
+	rules9 := &ResourceModifiers{
+		Version: "v1",
+		ResourceModifierRules: []ResourceModifierRule{
+			{
+				Conditions: Conditions{
+					GroupResource:     "deployments.apps",
+					ResourceNameRegex: "^test-.*$",
+					Namespaces:        []string{"bar", "foo"},
+				},
+				Patches: []JSONPatch{
+					{
+						Operation: "replace",
+						Path:      "/value/bool",
+						Value:     `"true"`,
+					},
+				},
+			},
+		},
+	}
+	cm10 := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-configmap",
+			Namespace: "test-namespace",
+		},
+		Data: map[string]string{
+			"sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n    groupResource: deployments.apps\n    resourceNameRegex: \"^test-.*$\"\n    namespaces:\n    - bar\n    - foo\n  patches:\n  - operation: replace\n    path: \"/value/bool\"\n    value: \"true\"\n\n\n",
+		},
+	}
+
+	rules10 := &ResourceModifiers{
+		Version: "v1",
+		ResourceModifierRules: []ResourceModifierRule{
+			{
+				Conditions: Conditions{
+					GroupResource:     "deployments.apps",
+					ResourceNameRegex: "^test-.*$",
+					Namespaces:        []string{"bar", "foo"},
+				},
+				Patches: []JSONPatch{
+					{
+						Operation: "replace",
+						Path:      "/value/bool",
+						Value:     "true",
+					},
+				},
+			},
+		},
+	}
 
 	type args struct {
 		cm *v1.ConfigMap
@@ -338,6 +396,22 @@ func TestGetResourceModifiersFromConfig(t *testing.T) {
 			want:    rules8,
 			wantErr: false,
 		},
+		{
+			name: "bool value as string",
+			args: args{
+				cm: cm9,
+			},
+			want:    rules9,
+			wantErr: false,
+		},
+		{
+			name: "bool value as bool",
+			args: args{
+				cm: cm10,
+			},
+			want:    rules10,
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -480,7 +554,24 @@ func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) {
 			},
 		},
 	}
-
+	cmTrue := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "v1",
+			"kind":       "ConfigMap",
+			"data": map[string]interface{}{
+				"test": "true",
+			},
+		},
+	}
+	cmFalse := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "v1",
+			"kind":       "ConfigMap",
+			"data": map[string]interface{}{
+				"test": "false",
+			},
+		},
+	}
 	type fields struct {
 		Version               string
 		ResourceModifierRules []ResourceModifierRule
@@ -496,6 +587,33 @@ func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) {
 		wantErr bool
 		wantObj *unstructured.Unstructured
 	}{
+		{
+			name: "configmap true false string",
+			fields: fields{
+				Version: "v1",
+				ResourceModifierRules: []ResourceModifierRule{
+					{
+						Conditions: Conditions{
+							GroupResource:     "configmaps",
+							ResourceNameRegex: ".*",
+						},
+						Patches: []JSONPatch{
+							{
+								Operation: "replace",
+								Path:      "/data/test",
+								Value:     `"false"`,
+							},
+						},
+					},
+				},
+			},
+			args: args{
+				obj:           cmTrue.DeepCopy(),
+				groupResource: "configmaps",
+			},
+			wantErr: false,
+			wantObj: cmFalse.DeepCopy(),
+		},
 		{
 			name: "Invalid Regex throws error",
 			fields: fields{

pkg/apis/velero/shared/uploader_config.go

@@ -0,0 +1,23 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package shared
+
+// UploaderConfig defines the configuration for the uploader.
+type UploaderConfig struct {
+	// ParallelFilesUpload is the number of files parallel uploads to perform when using the uploader.
+	ParallelFilesUpload int `json:"parallelFilesUpload,omitempty"`
+}

pkg/apis/velero/v1/backup_types.go

@@ -19,6 +19,8 @@ package v1
 import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 )
 
 type Metadata struct {
@@ -175,6 +177,11 @@ type BackupSpec struct {
 	// If DataMover is "" or "velero", the built-in data mover will be used.
 	// +optional
 	DataMover string `json:"datamover,omitempty"`
+
+	// UploaderConfig specifies the configuration for the uploader.
+	// +optional
+	// +nullable
+	UploaderConfig shared.UploaderConfig `json:"uploaderConfig,omitempty"`
 }
 
 // BackupHooks contains custom behaviors that should be executed at different phases of the backup.

pkg/apis/velero/v1/pod_volume_backup_types.go

@@ -51,6 +51,9 @@ type PodVolumeBackupSpec struct {
 	// volume backup as tags.
 	// +optional
 	Tags map[string]string `json:"tags,omitempty"`
+
+	// UploaderConfig specifies the configuration for the uploader.
+	UploaderConfig shared.UploaderConfig `json:"uploaderConfig,omitempty"`
 }
 
 // PodVolumeBackupPhase represents the lifecycle phase of a PodVolumeBackup.

pkg/apis/velero/v1/zz_generated.deepcopy.go

@@ -381,6 +381,7 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	out.UploaderConfig = in.UploaderConfig
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec.
@@ -951,6 +952,7 @@ func (in *PodVolumeBackupSpec) DeepCopyInto(out *PodVolumeBackupSpec) {
 			(*out)[key] = val
 		}
 	}
+	out.UploaderConfig = in.UploaderConfig
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeBackupSpec.

pkg/apis/velero/v2alpha1/data_upload_types.go

@@ -60,6 +60,9 @@ type DataUploadSpec struct {
 	// OperationTimeout specifies the time used to wait internal operations,
 	// before returning error as timeout.
 	OperationTimeout metav1.Duration `json:"operationTimeout"`
+
+	// UploaderConfig specifies the configuration for the uploader.
+	UploaderConfig shared.UploaderConfig `json:"uploaderConfig,omitempty"`
 }
 
 type SnapshotType string

pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go

@@ -236,6 +236,7 @@ func (in *DataUploadSpec) DeepCopyInto(out *DataUploadSpec) {
 		}
 	}
 	out.OperationTimeout = in.OperationTimeout
+	out.UploaderConfig = in.UploaderConfig
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadSpec.

pkg/backup/backup_test.go

@@ -71,6 +71,7 @@ func TestBackedUpItemsMatchesTarballContents(t *testing.T) {
 	req := &Request{
 		Backup:           defaultBackup().Result(),
 		SkippedPVTracker: NewSkipPVTracker(),
+		PVMap:            map[string]PvcPvInfo{},
 	}
 	backupFile := bytes.NewBuffer([]byte{})
 
@@ -84,8 +85,8 @@ func TestBackedUpItemsMatchesTarballContents(t *testing.T) {
 			builder.ForDeployment("zoo", "raz").Result(),
 		),
 		test.PVs(
-			builder.ForPersistentVolume("bar").Result(),
-			builder.ForPersistentVolume("baz").Result(),
+			builder.ForPersistentVolume("bar").ClaimRef("foo", "pvc1").Result(),
+			builder.ForPersistentVolume("baz").ClaimRef("bar", "pvc2").Result(),
 		),
 	}
 	for _, resource := range apiResources {

pkg/backup/item_backupper.go

@@ -250,6 +250,10 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti
 	namespace = metadata.GetNamespace()
 
 	if groupResource == kuberesource.PersistentVolumes {
+		if err := ib.addVolumeInfo(obj, log); err != nil {
+			backupErrs = append(backupErrs, err)
+		}
+
 		if err := ib.takePVSnapshot(obj, log); err != nil {
 			backupErrs = append(backupErrs, err)
 		}
@@ -685,6 +689,39 @@ func (ib *itemBackupper) unTrackSkippedPV(obj runtime.Unstructured, groupResourc
 	}
 }
 
+func (ib *itemBackupper) addVolumeInfo(obj runtime.Unstructured, log logrus.FieldLogger) error {
+	pv := new(corev1api.PersistentVolume)
+	err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pv)
+	if err != nil {
+		log.WithError(err).Warnf("Fail to convert PV")
+		return err
+	}
+
+	if ib.backupRequest.PVMap == nil {
+		ib.backupRequest.PVMap = make(map[string]PvcPvInfo)
+	}
+
+	pvcName := ""
+	pvcNamespace := ""
+	if pv.Spec.ClaimRef != nil {
+		pvcName = pv.Spec.ClaimRef.Name
+		pvcNamespace = pv.Spec.ClaimRef.Namespace
+
+		ib.backupRequest.PVMap[pvcNamespace+"/"+pvcName] = PvcPvInfo{
+			PVCName:      pvcName,
+			PVCNamespace: pvcNamespace,
+			PV:           *pv,
+		}
+	}
+
+	ib.backupRequest.PVMap[pv.Name] = PvcPvInfo{
+		PVCName:      pvcName,
+		PVCNamespace: pvcNamespace,
+		PV:           *pv,
+	}
+	return nil
+}
+
 // convert the input object to PV/PVC and get the PV name
 func getPVName(obj runtime.Unstructured, groupResource schema.GroupResource) (string, error) {
 	if groupResource == kuberesource.PersistentVolumes {

pkg/backup/item_backupper_test.go

@@ -19,10 +19,12 @@ package backup
 import (
 	"testing"
 
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
 	"github.com/vmware-tanzu/velero/pkg/kuberesource"
+	"github.com/vmware-tanzu/velero/pkg/volume"
 
 	"github.com/stretchr/testify/assert"
 	corev1api "k8s.io/api/core/v1"
@@ -237,3 +239,55 @@ func TestRandom(t *testing.T) {
 	err2 := runtime.DefaultUnstructuredConverter.FromUnstructured(o, pvc)
 	t.Logf("err1: %v, err2: %v", err1, err2)
 }
+
+func TestAddVolumeInfo(t *testing.T) {
+	tests := []struct {
+		name               string
+		pv                 *corev1api.PersistentVolume
+		expectedVolumeInfo map[string]PvcPvInfo
+	}{
+		{
+			name: "PV has ClaimRef",
+			pv:   builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(),
+			expectedVolumeInfo: map[string]PvcPvInfo{
+				"testPV": {
+					PVCName:      "testPVC",
+					PVCNamespace: "testNS",
+					PV:           *builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(),
+				},
+				"testNS/testPVC": {
+					PVCName:      "testPVC",
+					PVCNamespace: "testNS",
+					PV:           *builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(),
+				},
+			},
+		},
+		{
+			name: "PV has no ClaimRef",
+			pv:   builder.ForPersistentVolume("testPV").Result(),
+			expectedVolumeInfo: map[string]PvcPvInfo{
+				"testPV": {
+					PVCName:      "",
+					PVCNamespace: "",
+					PV:           *builder.ForPersistentVolume("testPV").Result(),
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ib := itemBackupper{}
+			ib.backupRequest = new(Request)
+			ib.backupRequest.VolumeInfos.VolumeInfos = make([]volume.VolumeInfo, 0)
+
+			pvObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pv)
+			require.NoError(t, err)
+			logger := logrus.StandardLogger()
+
+			err = ib.addVolumeInfo(&unstructured.Unstructured{Object: pvObj}, logger)
+			require.NoError(t, err)
+			require.Equal(t, tc.expectedVolumeInfo, ib.backupRequest.PVMap)
+		})
+	}
+}

pkg/backup/pv_skip_tracker.go

@@ -10,6 +10,14 @@ type SkippedPV struct {
 	Reasons []PVSkipReason `json:"reasons"`
 }
 
+func (s *SkippedPV) SerializeSkipReasons() string {
+	ret := ""
+	for _, reason := range s.Reasons {
+		ret = ret + reason.Approach + ": " + reason.Reason + ";"
+	}
+	return ret
+}
+
 type PVSkipReason struct {
 	Approach string `json:"approach"`
 	Reason   string `json:"reason"`

pkg/backup/pv_skip_tracker_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestSummary(t *testing.T) {
@@ -41,3 +42,14 @@ func TestSummary(t *testing.T) {
 	}
 	assert.Equal(t, expected, tracker.Summary())
 }
+
+func TestSerializeSkipReasons(t *testing.T) {
+	tracker := NewSkipPVTracker()
+	//tracker.Track("pv5", "", "skipped due to policy")
+	tracker.Track("pv3", podVolumeApproach, "it's set to opt-out")
+	tracker.Track("pv3", csiSnapshotApproach, "not applicable for CSI ")
+
+	for _, skippedPV := range tracker.Summary() {
+		require.Equal(t, "csiSnapshot: not applicable for CSI ;podvolume: it's set to opt-out;", skippedPV.SerializeSkipReasons())
+	}
+}

pkg/backup/request.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"sort"
 
+	corev1api "k8s.io/api/core/v1"
+
 	"github.com/vmware-tanzu/velero/internal/hook"
 	"github.com/vmware-tanzu/velero/internal/resourcepolicies"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -52,6 +54,16 @@ type Request struct {
 	itemOperationsList        *[]*itemoperation.BackupOperation
 	ResPolicies               *resourcepolicies.Policies
 	SkippedPVTracker          *skipPVTracker
+	// A map contains the backup-included PV detail content.
+	// The key is PV name or PVC name(The format is PVC-namespace/PVC-name)
+	PVMap       map[string]PvcPvInfo
+	VolumeInfos volume.VolumeInfos
+}
+
+type PvcPvInfo struct {
+	PVCName      string
+	PVCNamespace string
+	PV           corev1api.PersistentVolume
 }
 
 // GetItemOperationsList returns ItemOperationsList, initializing it if necessary

pkg/backup/snapshots.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
-	snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/sets"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -17,27 +16,25 @@ import (
 
 // Common function to update the status of CSI snapshots
 // returns VolumeSnapshot, VolumeSnapshotContent, VolumeSnapshotClasses referenced
-func UpdateBackupCSISnapshotsStatus(client kbclient.Client, volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, backup *velerov1api.Backup, backupLog logrus.FieldLogger) (volumeSnapshots []snapshotv1api.VolumeSnapshot, volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass) {
+func UpdateBackupCSISnapshotsStatus(client kbclient.Client, globalCRClient kbclient.Client, backup *velerov1api.Backup, backupLog logrus.FieldLogger) (volumeSnapshots []snapshotv1api.VolumeSnapshot, volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass) {
 	if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) {
 		backupLog.Info("backup SnapshotMoveData is set to true, skip VolumeSnapshot resource persistence.")
 	} else if features.IsEnabled(velerov1api.CSIFeatureFlag) {
 		selector := label.NewSelectorForBackup(backup.Name)
 		vscList := &snapshotv1api.VolumeSnapshotContentList{}
 
-		if volumeSnapshotLister != nil {
-			tmpVSs, err := volumeSnapshotLister.List(label.NewSelectorForBackup(backup.Name))
-			if err != nil {
-				backupLog.Error(err)
-			}
-			for _, vs := range tmpVSs {
-				volumeSnapshots = append(volumeSnapshots, *vs)
-			}
-		}
-
-		err := client.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector})
+		vsList := new(snapshotv1api.VolumeSnapshotList)
+		err := globalCRClient.List(context.TODO(), vsList, &kbclient.ListOptions{
+			LabelSelector: label.NewSelectorForBackup(backup.Name),
+		})
 		if err != nil {
 			backupLog.Error(err)
 		}
+		volumeSnapshots = append(volumeSnapshots, vsList.Items...)
+
+		if err := client.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector}); err != nil {
+			backupLog.Error(err)
+		}
 		if len(vscList.Items) >= 0 {
 			volumeSnapshotContents = vscList.Items
 		}

pkg/builder/backup_builder.go

@@ -300,6 +300,12 @@ func (b *BackupBuilder) DataMover(name string) *BackupBuilder {
 	return b
 }
 
+// ParallelFilesUpload sets the Backup's uploader parallel uploads
+func (b *BackupBuilder) ParallelFilesUpload(parallel int) *BackupBuilder {
+	b.object.Spec.UploaderConfig.ParallelFilesUpload = parallel
+	return b
+}
+
 // WithStatus sets the Backup's status.
 func (b *BackupBuilder) WithStatus(status velerov1api.BackupStatus) *BackupBuilder {
 	b.object.Status = status

pkg/builder/volume_snapshot_builder.go

@@ -18,6 +18,7 @@ package builder
 
 import (
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -68,7 +69,21 @@ func (v *VolumeSnapshotBuilder) BoundVolumeSnapshotContentName(vscName string) *
 	return v
 }
 
+// SourcePVC set the built VolumeSnapshot's spec.Source.PersistentVolumeClaimName.
 func (v *VolumeSnapshotBuilder) SourcePVC(name string) *VolumeSnapshotBuilder {
 	v.object.Spec.Source.PersistentVolumeClaimName = &name
 	return v
 }
+
+// RestoreSize set the built VolumeSnapshot's status.RestoreSize.
+func (v *VolumeSnapshotBuilder) RestoreSize(size string) *VolumeSnapshotBuilder {
+	resourceSize := resource.MustParse(size)
+	v.object.Status.RestoreSize = &resourceSize
+	return v
+}
+
+// VolumeSnapshotClass set the built VolumeSnapshot's spec.VolumeSnapshotClassName value.
+func (v *VolumeSnapshotBuilder) VolumeSnapshotClass(name string) *VolumeSnapshotBuilder {
+	v.object.Spec.VolumeSnapshotClassName = &name
+	return v
+}

pkg/builder/volume_snapshot_content_builder.go

@@ -59,6 +59,7 @@ func (v *VolumeSnapshotContentBuilder) DeletionPolicy(policy snapshotv1api.Delet
 	return v
 }
 
+// VolumeSnapshotRef sets the built VolumeSnapshotContent's spec.VolumeSnapshotRef value.
 func (v *VolumeSnapshotContentBuilder) VolumeSnapshotRef(namespace, name string) *VolumeSnapshotContentBuilder {
 	v.object.Spec.VolumeSnapshotRef = v1.ObjectReference{
 		APIVersion: "snapshot.storage.k8s.io/v1",
@@ -68,3 +69,18 @@ func (v *VolumeSnapshotContentBuilder) VolumeSnapshotRef(namespace, name string)
 	}
 	return v
 }
+
+// VolumeSnapshotClassName sets the built VolumeSnapshotContent's spec.VolumeSnapshotClassName value.
+func (v *VolumeSnapshotContentBuilder) VolumeSnapshotClassName(name string) *VolumeSnapshotContentBuilder {
+	v.object.Spec.VolumeSnapshotClassName = &name
+	return v
+}
+
+// ObjectMeta applies functional options to the VolumeSnapshotContent's ObjectMeta.
+func (v *VolumeSnapshotContentBuilder) ObjectMeta(opts ...ObjectMetaOpt) *VolumeSnapshotContentBuilder {
+	for _, opt := range opts {
+		opt(v.object)
+	}
+
+	return v
+}

pkg/client/factory.go

@@ -24,6 +24,7 @@ import (
 	k8scheme "k8s.io/client-go/kubernetes/scheme"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -158,6 +159,9 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) {
 	if err := apiextv1.AddToScheme(scheme); err != nil {
 		return nil, err
 	}
+	if err := snapshotv1api.AddToScheme(scheme); err != nil {
+		return nil, err
+	}
 	kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{
 		Scheme: scheme,
 	})

pkg/cmd/cli/backup/create.go

@@ -106,6 +106,7 @@ type CreateOptions struct {
 	ItemOperationTimeout            time.Duration
 	ResPoliciesConfigmap            string
 	client                          kbclient.WithWatch
+	ParallelFilesUpload             int
 }
 
 func NewCreateOptions() *CreateOptions {
@@ -151,6 +152,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
 
 	flags.StringVar(&o.ResPoliciesConfigmap, "resource-policies-configmap", "", "Reference to the resource policies configmap that backup using")
 	flags.StringVar(&o.DataMover, "data-mover", "", "Specify the data mover to be used by the backup. If the parameter is not set or set as 'velero', the built-in data mover will be used")
+	flags.IntVar(&o.ParallelFilesUpload, "parallel-files-upload", 0, "Number of files uploads simultaneously when running a backup. This is only applicable for the kopia uploader")
 }
 
 // BindWait binds the wait flag separately so it is not called by other create
@@ -396,6 +398,9 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro
 		if o.ResPoliciesConfigmap != "" {
 			backupBuilder.ResourcePolicies(o.ResPoliciesConfigmap)
 		}
+		if o.ParallelFilesUpload > 0 {
+			backupBuilder.ParallelFilesUpload(o.ParallelFilesUpload)
+		}
 	}
 
 	backup := backupBuilder.ObjectMeta(builder.WithLabelsMap(o.Labels.Data())).Result()

pkg/cmd/cli/backup/create_test.go

@@ -185,7 +185,7 @@ func TestCreateCommand(t *testing.T) {
 		defaultVolumesToFsBackup := "true"
 		resPoliciesConfigmap := "cm-name-2"
 		dataMover := "velero"
-
+		parallelFilesUpload := 10
 		flags := new(flag.FlagSet)
 		o := NewCreateOptions()
 		o.BindFlags(flags)
@@ -213,6 +213,7 @@ func TestCreateCommand(t *testing.T) {
 		flags.Parse([]string{"--default-volumes-to-fs-backup", defaultVolumesToFsBackup})
 		flags.Parse([]string{"--resource-policies-configmap", resPoliciesConfigmap})
 		flags.Parse([]string{"--data-mover", dataMover})
+		flags.Parse([]string{"--parallel-files-upload", fmt.Sprintf("%d", parallelFilesUpload)})
 		//flags.Parse([]string{"--wait"})
 
 		client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch)
@@ -261,6 +262,7 @@ func TestCreateCommand(t *testing.T) {
 		require.Equal(t, defaultVolumesToFsBackup, o.DefaultVolumesToFsBackup.String())
 		require.Equal(t, resPoliciesConfigmap, o.ResPoliciesConfigmap)
 		require.Equal(t, dataMover, o.DataMover)
+		require.Equal(t, parallelFilesUpload, o.ParallelFilesUpload)
 		//assert.Equal(t, true, o.Wait)
 
 		// verify oldAndNewFilterParametersUsedTogether

pkg/cmd/server/server.go

@@ -23,21 +23,16 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
-	"reflect"
 	"strings"
 	"time"
 
 	logrusr "github.com/bombsimon/logrusr/v3"
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
-	snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
-	snapshotv1informers "github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions"
-	snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	corev1api "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -244,15 +239,17 @@ func NewCommand(f client.Factory) *cobra.Command {
 }
 
 type server struct {
-	namespace             string
-	metricsAddress        string
-	kubeClientConfig      *rest.Config
-	kubeClient            kubernetes.Interface
-	discoveryClient       discovery.DiscoveryInterface
-	discoveryHelper       velerodiscovery.Helper
-	dynamicClient         dynamic.Interface
-	csiSnapshotClient     *snapshotv1client.Clientset
-	csiSnapshotLister     snapshotv1listers.VolumeSnapshotLister
+	namespace        string
+	metricsAddress   string
+	kubeClientConfig *rest.Config
+	kubeClient       kubernetes.Interface
+	discoveryClient  discovery.DiscoveryInterface
+	discoveryHelper  velerodiscovery.Helper
+	dynamicClient    dynamic.Interface
+	// controller-runtime client. the difference from the controller-manager's client
+	// is that the the controller-manager's client is limited to list namespaced-scoped
+	// resources in the namespace where Velero is installed, or the cluster-scoped
+	// resources. The crClient doesn't have the limitation.
 	crClient              ctrlclient.Client
 	ctx                   context.Context
 	cancelFunc            context.CancelFunc
@@ -399,23 +396,6 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 		featureVerifier:       featureVerifier,
 	}
 
-	// Setup CSI snapshot client and lister
-	var csiSnapClient *snapshotv1client.Clientset
-	if features.IsEnabled(velerov1api.CSIFeatureFlag) {
-		csiSnapClient, err = snapshotv1client.NewForConfig(clientConfig)
-		if err != nil {
-			cancelFunc()
-			return nil, err
-		}
-		s.csiSnapshotClient = csiSnapClient
-
-		s.csiSnapshotLister, err = s.getCSIVolumeSnapshotListers()
-		if err != nil {
-			cancelFunc()
-			return nil, err
-		}
-	}
-
 	return s, nil
 }
 
@@ -541,7 +521,7 @@ High priorities:
   - PVs go before PVCs because PVCs depend on them.
   - PVCs go before pods or controllers so they can be mounted as volumes.
   - Service accounts go before secrets so service account token secrets can be filled automatically.
-  - Secrets and config maps go before pods or controllers so they can be mounted
+  - Secrets and ConfigMaps go before pods or controllers so they can be mounted
     as volumes.
   - Limit ranges go before pods or controllers so pods can use them.
   - Pods go before controllers so they can be explicitly restored and potentially
@@ -615,40 +595,6 @@ func (s *server) initRepoManager() error {
 	return nil
 }
 
-func (s *server) getCSIVolumeSnapshotListers() (vsLister snapshotv1listers.VolumeSnapshotLister, err error) {
-	_, err = s.discoveryClient.ServerResourcesForGroupVersion(snapshotv1api.SchemeGroupVersion.String())
-	switch {
-	case apierrors.IsNotFound(err):
-		// CSI is enabled, but the required CRDs aren't installed, so halt.
-		s.logger.Warnf("The '%s' feature flag was specified, but CSI API group [%s] was not found.", velerov1api.CSIFeatureFlag, snapshotv1api.SchemeGroupVersion.String())
-		err = nil
-	case err == nil:
-		wrapper := NewCSIInformerFactoryWrapper(s.csiSnapshotClient)
-
-		s.logger.Debug("Creating CSI listers")
-		// Access the wrapped factory directly here since we've already done the feature flag check above to know it's safe.
-		vsLister = wrapper.factory.Snapshot().V1().VolumeSnapshots().Lister()
-
-		// start the informers & and wait for the caches to sync
-		wrapper.Start(s.ctx.Done())
-		s.logger.Info("Waiting for informer caches to sync")
-		csiCacheSyncResults := wrapper.WaitForCacheSync(s.ctx.Done())
-		s.logger.Info("Done waiting for informer caches to sync")
-
-		for informer, synced := range csiCacheSyncResults {
-			if !synced {
-				err = errors.Errorf("cache was not synced for informer %v", informer)
-				return
-			}
-			s.logger.WithField("informer", informer).Info("Informer cache synced")
-		}
-	case err != nil:
-		s.logger.Errorf("fail to find snapshot v1 schema: %s", err)
-	}
-
-	return vsLister, err
-}
-
 func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string) error {
 	s.logger.Info("Starting controllers")
 
@@ -775,10 +721,10 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			s.metrics,
 			backupStoreGetter,
 			s.config.formatFlag.Parse(),
-			s.csiSnapshotLister,
 			s.credentialFileStore,
 			s.config.maxConcurrentK8SConnections,
 			s.config.defaultSnapshotMoveData,
+			s.crClient,
 		).SetupWithManager(s.mgr); err != nil {
 			s.logger.Fatal(err, "unable to create controller", "controller", controller.Backup)
 		}
@@ -837,7 +783,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 		cmd.CheckError(err)
 		r := controller.NewBackupFinalizerReconciler(
 			s.mgr.GetClient(),
-			s.csiSnapshotLister,
+			s.crClient,
 			clock.RealClock{},
 			backupper,
 			newPluginManager,
@@ -1027,37 +973,6 @@ func (s *server) runProfiler() {
 	}
 }
 
-// CSIInformerFactoryWrapper is a proxy around the CSI SharedInformerFactory that checks the CSI feature flag before performing operations.
-type CSIInformerFactoryWrapper struct {
-	factory snapshotv1informers.SharedInformerFactory
-}
-
-func NewCSIInformerFactoryWrapper(c snapshotv1client.Interface) *CSIInformerFactoryWrapper {
-	// If no namespace is specified, all namespaces are watched.
-	// This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer
-	w := &CSIInformerFactoryWrapper{}
-
-	if features.IsEnabled(velerov1api.CSIFeatureFlag) {
-		w.factory = snapshotv1informers.NewSharedInformerFactoryWithOptions(c, 0)
-	}
-	return w
-}
-
-// Start proxies the Start call to the CSI SharedInformerFactory.
-func (w *CSIInformerFactoryWrapper) Start(stopCh <-chan struct{}) {
-	if features.IsEnabled(velerov1api.CSIFeatureFlag) {
-		w.factory.Start(stopCh)
-	}
-}
-
-// WaitForCacheSync proxies the WaitForCacheSync call to the CSI SharedInformerFactory.
-func (w *CSIInformerFactoryWrapper) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool {
-	if features.IsEnabled(velerov1api.CSIFeatureFlag) {
-		return w.factory.WaitForCacheSync(stopCh)
-	}
-	return nil
-}
-
 // if there is a restarting during the reconciling of backups/restores/etc, these CRs may be stuck in progress status
 // markInProgressCRsFailed tries to mark the in progress CRs as failed when starting the server to avoid the issue
 func markInProgressCRsFailed(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, log logrus.FieldLogger) {
@@ -1081,13 +996,13 @@ func markInProgressBackupsFailed(ctx context.Context, client ctrlclient.Client,
 	}
 
 	for i, backup := range backups.Items {
-		if backup.Status.Phase != velerov1api.BackupPhaseInProgress && backup.Status.Phase != velerov1api.BackupPhaseWaitingForPluginOperations {
+		if backup.Status.Phase != velerov1api.BackupPhaseInProgress {
 			log.Debugf("the status of backup %q is %q, skip", backup.GetName(), backup.Status.Phase)
 			continue
 		}
 		updated := backup.DeepCopy()
 		updated.Status.Phase = velerov1api.BackupPhaseFailed
-		updated.Status.FailureReason = fmt.Sprintf("found a backup with status %q during the server starting, mark it as %q", velerov1api.BackupPhaseInProgress, updated.Status.Phase)
+		updated.Status.FailureReason = fmt.Sprintf("found a backup with status %q during the server starting, mark it as %q", backup.Status.Phase, updated.Status.Phase)
 		updated.Status.CompletionTimestamp = &metav1.Time{Time: time.Now()}
 		if err := client.Patch(ctx, updated, ctrlclient.MergeFrom(&backups.Items[i])); err != nil {
 			log.WithError(errors.WithStack(err)).Errorf("failed to patch backup %q", backup.GetName())
@@ -1105,13 +1020,13 @@ func markInProgressRestoresFailed(ctx context.Context, client ctrlclient.Client,
 		return
 	}
 	for i, restore := range restores.Items {
-		if restore.Status.Phase != velerov1api.RestorePhaseInProgress && restore.Status.Phase != velerov1api.RestorePhaseWaitingForPluginOperations {
+		if restore.Status.Phase != velerov1api.RestorePhaseInProgress {
 			log.Debugf("the status of restore %q is %q, skip", restore.GetName(), restore.Status.Phase)
 			continue
 		}
 		updated := restore.DeepCopy()
 		updated.Status.Phase = velerov1api.RestorePhaseFailed
-		updated.Status.FailureReason = fmt.Sprintf("found a restore with status %q during the server starting, mark it as %q", velerov1api.RestorePhaseInProgress, updated.Status.Phase)
+		updated.Status.FailureReason = fmt.Sprintf("found a restore with status %q during the server starting, mark it as %q", restore.Status.Phase, updated.Status.Phase)
 		updated.Status.CompletionTimestamp = &metav1.Time{Time: time.Now()}
 		if err := client.Patch(ctx, updated, ctrlclient.MergeFrom(&restores.Items[i])); err != nil {
 			log.WithError(errors.WithStack(err)).Errorf("failed to patch restore %q", restore.GetName())
@@ -1134,7 +1049,9 @@ func markDataUploadsCancel(ctx context.Context, client ctrlclient.Client, backup
 		du := dataUploads.Items[i]
 		if du.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted ||
 			du.Status.Phase == velerov2alpha1api.DataUploadPhasePrepared ||
-			du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress {
+			du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress ||
+			du.Status.Phase == velerov2alpha1api.DataUploadPhaseNew ||
+			du.Status.Phase == "" {
 			err := controller.UpdateDataUploadWithRetry(ctx, client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, log.WithField("dataupload", du.Name),
 				func(dataUpload *velerov2alpha1api.DataUpload) {
 					dataUpload.Spec.Cancel = true
@@ -1162,7 +1079,9 @@ func markDataDownloadsCancel(ctx context.Context, client ctrlclient.Client, rest
 		dd := dataDownloads.Items[i]
 		if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted ||
 			dd.Status.Phase == velerov2alpha1api.DataDownloadPhasePrepared ||
-			dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress {
+			dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress ||
+			dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseNew ||
+			dd.Status.Phase == "" {
 			err := controller.UpdateDataDownloadWithRetry(ctx, client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log.WithField("datadownload", dd.Name),
 				func(dataDownload *velerov2alpha1api.DataDownload) {
 					dataDownload.Spec.Cancel = true

pkg/cmd/util/output/backup_describer.go

@@ -86,6 +86,11 @@ func DescribeBackup(
 			DescribeResourcePolicies(d, backup.Spec.ResourcePolicy)
 		}
 
+		if backup.Spec.UploaderConfig.ParallelFilesUpload > 0 {
+			d.Println()
+			DescribeUploaderConfig(d, backup.Spec)
+		}
+
 		status := backup.Status
 		if len(status.ValidationErrors) > 0 {
 			d.Println()
@@ -111,13 +116,19 @@ func DescribeBackup(
 	})
 }
 
-// DescribeResourcePolicies describes resource policiesin human-readable format
+// DescribeResourcePolicies describes resource policies in human-readable format
 func DescribeResourcePolicies(d *Describer, resPolicies *v1.TypedLocalObjectReference) {
 	d.Printf("Resource policies:\n")
 	d.Printf("\tType:\t%s\n", resPolicies.Kind)
 	d.Printf("\tName:\t%s\n", resPolicies.Name)
 }
 
+// DescribeUploaderConfig describes uploader config in human-readable format
+func DescribeUploaderConfig(d *Describer, spec velerov1api.BackupSpec) {
+	d.Printf("Uploader config:\n")
+	d.Printf("\tParallel files upload:\t%d\n", spec.UploaderConfig.ParallelFilesUpload)
+}
+
 // DescribeBackupSpec describes a backup spec in human-readable format.
 func DescribeBackupSpec(d *Describer, spec velerov1api.BackupSpec) {
 	// TODO make a helper for this and use it in all the describers.

pkg/cmd/util/output/backup_describer_test.go

@@ -20,6 +20,22 @@ import (
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 )
 
+func TestDescribeUploaderConfig(t *testing.T) {
+	input := builder.ForBackup("test-ns", "test-backup-1").ParallelFilesUpload(10).Result().Spec
+	d := &Describer{
+		Prefix: "",
+		out:    &tabwriter.Writer{},
+		buf:    &bytes.Buffer{},
+	}
+	d.out.Init(d.buf, 0, 8, 2, ' ', 0)
+	DescribeUploaderConfig(d, input)
+	d.out.Flush()
+	expect := `Uploader config:
+  Parallel files upload:  10
+`
+	assert.Equal(t, expect, d.buf.String())
+}
+
 func TestDescribeResourcePolicies(t *testing.T) {
 	input := &v1.TypedLocalObjectReference{
 		Kind: "configmap",

pkg/cmd/util/output/schedule_describe_test.go

@@ -59,7 +59,7 @@ Last Backup:  <never>
 	input2 := builder.ForSchedule("velero", "schedule-2").
 		Phase(velerov1api.SchedulePhaseEnabled).
 		CronSchedule("0 0 * * *").
-		Template(builder.ForBackup("velero", "backup-1").Result().Spec).
+		Template(builder.ForBackup("velero", "backup-1").ParallelFilesUpload(10).Result().Spec).
 		LastBackupTime("2023-06-25 15:04:05").Result()
 	expect2 := `Name:         schedule-2
 Namespace:    velero
@@ -68,6 +68,9 @@ Annotations:  <none>
 
 Phase:  Enabled
 
+Uploader config:
+  Parallel files upload:  10
+
 Paused:  false
 
 Schedule:  0 0 * * *

pkg/cmd/util/output/schedule_describer.go

@@ -48,6 +48,11 @@ func DescribeSchedule(schedule *v1.Schedule) string {
 			DescribeResourcePolicies(d, schedule.Spec.Template.ResourcePolicy)
 		}
 
+		if schedule.Spec.Template.UploaderConfig.ParallelFilesUpload > 0 {
+			d.Println()
+			DescribeUploaderConfig(d, schedule.Spec.Template)
+		}
+
 		status := schedule.Status
 		if len(status.ValidationErrors) > 0 {
 			d.Println()

pkg/controller/backup_controller.go

@@ -21,12 +21,11 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
-	snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
-	snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	corev1api "k8s.io/api/core/v1"
@@ -43,14 +42,18 @@ import (
 	"github.com/vmware-tanzu/velero/internal/resourcepolicies"
 	"github.com/vmware-tanzu/velero/internal/storage"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
 	pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/discovery"
 	"github.com/vmware-tanzu/velero/pkg/features"
+	"github.com/vmware-tanzu/velero/pkg/itemoperation"
+	"github.com/vmware-tanzu/velero/pkg/kuberesource"
 	"github.com/vmware-tanzu/velero/pkg/label"
 	"github.com/vmware-tanzu/velero/pkg/metrics"
 	"github.com/vmware-tanzu/velero/pkg/persistence"
 	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
 	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
+	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 	"github.com/vmware-tanzu/velero/pkg/util/encode"
@@ -84,11 +87,10 @@ type backupReconciler struct {
 	metrics                     *metrics.ServerMetrics
 	backupStoreGetter           persistence.ObjectBackupStoreGetter
 	formatFlag                  logging.Format
-	volumeSnapshotLister        snapshotv1listers.VolumeSnapshotLister
-	volumeSnapshotClient        snapshotterClientSet.Interface
 	credentialFileStore         credentials.FileStore
 	maxConcurrentK8SConnections int
 	defaultSnapshotMoveData     bool
+	globalCRClient              kbclient.Client
 }
 
 func NewBackupReconciler(
@@ -110,10 +112,10 @@ func NewBackupReconciler(
 	metrics *metrics.ServerMetrics,
 	backupStoreGetter persistence.ObjectBackupStoreGetter,
 	formatFlag logging.Format,
-	volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister,
 	credentialStore credentials.FileStore,
 	maxConcurrentK8SConnections int,
 	defaultSnapshotMoveData bool,
+	globalCRClient kbclient.Client,
 ) *backupReconciler {
 	b := &backupReconciler{
 		ctx:                         ctx,
@@ -135,10 +137,10 @@ func NewBackupReconciler(
 		metrics:                     metrics,
 		backupStoreGetter:           backupStoreGetter,
 		formatFlag:                  formatFlag,
-		volumeSnapshotLister:        volumeSnapshotLister,
 		credentialFileStore:         credentialStore,
 		maxConcurrentK8SConnections: maxConcurrentK8SConnections,
 		defaultSnapshotMoveData:     defaultSnapshotMoveData,
+		globalCRClient:              globalCRClient,
 	}
 	b.updateTotalBackupMetric()
 	return b
@@ -317,6 +319,7 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg
 	request := &pkgbackup.Request{
 		Backup:           backup.DeepCopy(), // don't modify items in the cache
 		SkippedPVTracker: pkgbackup.NewSkipPVTracker(),
+		PVMap:            map[string]pkgbackup.PvcPvInfo{},
 	}
 
 	// set backup major version - deprecated, use Status.FormatVersion
@@ -665,7 +668,7 @@ func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error {
 			backup.Status.VolumeSnapshotsCompleted++
 		}
 	}
-	volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses := pkgbackup.UpdateBackupCSISnapshotsStatus(b.kbClient, b.volumeSnapshotLister, backup.Backup, backupLog)
+	volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses := pkgbackup.UpdateBackupCSISnapshotsStatus(b.kbClient, b.globalCRClient, backup.Backup, backupLog)
 
 	// Iterate over backup item operations and update progress.
 	// Any errors on operations at this point should be added to backup errors.
@@ -734,6 +737,8 @@ func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error {
 	if logFile, err := backupLog.GetPersistFile(); err != nil {
 		fatalErrs = append(fatalErrs, errors.Wrap(err, "error getting backup log file"))
 	} else {
+		backup.VolumeInfos.VolumeInfos = generateVolumeInfo(backup, volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses, b.globalCRClient, backupLog)
+
 		if errs := persistBackup(backup, backupFile, logFile, backupStore, volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses, results); len(errs) > 0 {
 			fatalErrs = append(fatalErrs, errs...)
 		}
@@ -796,7 +801,6 @@ func persistBackup(backup *pkgbackup.Request,
 ) []error {
 	persistErrs := []error{}
 	backupJSON := new(bytes.Buffer)
-	volumeInfos := make([]volume.VolumeInfo, 0)
 
 	if err := encode.To(backup.Backup, "json", backupJSON); err != nil {
 		persistErrs = append(persistErrs, errors.Wrap(err, "error encoding backup"))
@@ -843,7 +847,7 @@ func persistBackup(backup *pkgbackup.Request,
 		persistErrs = append(persistErrs, errs...)
 	}
 
-	volumeInfoJSON, errs := encode.ToJSONGzip(volumeInfos, "backup volumes information")
+	volumeInfoJSON, errs := encode.ToJSONGzip(backup.VolumeInfos, "backup volumes information")
 	if errs != nil {
 		persistErrs = append(persistErrs, errs...)
 	}
@@ -908,3 +912,328 @@ func oldAndNewFilterParametersUsedTogether(backupSpec velerov1api.BackupSpec) bo
 
 	return haveOldResourceFilterParameters && haveNewResourceFilterParameters
 }
+
+func generateVolumeInfo(backup *pkgbackup.Request, csiVolumeSnapshots []snapshotv1api.VolumeSnapshot,
+	csiVolumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, csiVolumesnapshotClasses []snapshotv1api.VolumeSnapshotClass,
+	crClient kbclient.Client, logger logrus.FieldLogger) []volume.VolumeInfo {
+	volumeInfos := make([]volume.VolumeInfo, 0)
+
+	skippedVolumeInfos := generateVolumeInfoForSkippedPV(backup, logger)
+	volumeInfos = append(volumeInfos, skippedVolumeInfos...)
+
+	nativeSnapshotVolumeInfos := generateVolumeInfoForVeleroNativeSnapshot(backup, logger)
+	volumeInfos = append(volumeInfos, nativeSnapshotVolumeInfos...)
+
+	csiVolumeInfos := generateVolumeInfoForCSIVolumeSnapshot(backup, csiVolumeSnapshots, csiVolumeSnapshotContents, csiVolumesnapshotClasses, logger)
+	volumeInfos = append(volumeInfos, csiVolumeInfos...)
+
+	pvbVolumeInfos := generateVolumeInfoFromPVB(backup, crClient, logger)
+	volumeInfos = append(volumeInfos, pvbVolumeInfos...)
+
+	dataUploadVolumeInfos := generateVolumeInfoFromDataUpload(backup, crClient, logger)
+	volumeInfos = append(volumeInfos, dataUploadVolumeInfos...)
+
+	return volumeInfos
+}
+
+// generateVolumeInfoForSkippedPV generate VolumeInfos for SkippedPV.
+func generateVolumeInfoForSkippedPV(backup *pkgbackup.Request, logger logrus.FieldLogger) []volume.VolumeInfo {
+	tmpVolumeInfos := make([]volume.VolumeInfo, 0)
+
+	for _, skippedPV := range backup.SkippedPVTracker.Summary() {
+		if pvcPVInfo, ok := backup.PVMap[skippedPV.Name]; ok {
+			volumeInfo := volume.VolumeInfo{
+				PVCName:           pvcPVInfo.PVCName,
+				PVCNamespace:      pvcPVInfo.PVCNamespace,
+				PVName:            skippedPV.Name,
+				SnapshotDataMoved: false,
+				Skipped:           true,
+				SkippedReason:     skippedPV.SerializeSkipReasons(),
+				PVInfo: volume.PVInfo{
+					ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy),
+					Labels:        pvcPVInfo.PV.Labels,
+				},
+			}
+			tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo)
+		} else {
+			logger.Warnf("Cannot find info for PV %s", skippedPV.Name)
+			continue
+		}
+	}
+
+	return tmpVolumeInfos
+}
+
+// generateVolumeInfoForVeleroNativeSnapshot generate VolumeInfos for Velero native snapshot
+func generateVolumeInfoForVeleroNativeSnapshot(backup *pkgbackup.Request, logger logrus.FieldLogger) []volume.VolumeInfo {
+	tmpVolumeInfos := make([]volume.VolumeInfo, 0)
+
+	for _, nativeSnapshot := range backup.VolumeSnapshots {
+		var iops int64
+		if nativeSnapshot.Spec.VolumeIOPS != nil {
+			iops = *nativeSnapshot.Spec.VolumeIOPS
+		}
+
+		if pvcPVInfo, ok := backup.PVMap[nativeSnapshot.Spec.PersistentVolumeName]; ok {
+			volumeInfo := volume.VolumeInfo{
+				BackupMethod:      volume.NativeSnapshot,
+				PVCName:           pvcPVInfo.PVCName,
+				PVCNamespace:      pvcPVInfo.PVCNamespace,
+				PVName:            pvcPVInfo.PV.Name,
+				SnapshotDataMoved: false,
+				Skipped:           false,
+				NativeSnapshotInfo: volume.NativeSnapshotInfo{
+					SnapshotHandle: nativeSnapshot.Status.ProviderSnapshotID,
+					VolumeType:     nativeSnapshot.Spec.VolumeType,
+					VolumeAZ:       nativeSnapshot.Spec.VolumeAZ,
+					IOPS:           strconv.FormatInt(iops, 10),
+				},
+				PVInfo: volume.PVInfo{
+					ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy),
+					Labels:        pvcPVInfo.PV.Labels,
+				},
+			}
+
+			tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo)
+		} else {
+			logger.Warnf("cannot find info for PV %s", nativeSnapshot.Spec.PersistentVolumeName)
+			continue
+		}
+	}
+
+	return tmpVolumeInfos
+}
+
+// generateVolumeInfoForCSIVolumeSnapshot generate VolumeInfos for CSI VolumeSnapshot
+func generateVolumeInfoForCSIVolumeSnapshot(backup *pkgbackup.Request, csiVolumeSnapshots []snapshotv1api.VolumeSnapshot,
+	csiVolumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, csiVolumesnapshotClasses []snapshotv1api.VolumeSnapshotClass,
+	logger logrus.FieldLogger) []volume.VolumeInfo {
+	tmpVolumeInfos := make([]volume.VolumeInfo, 0)
+
+	for _, volumeSnapshot := range csiVolumeSnapshots {
+		var volumeSnapshotClass *snapshotv1api.VolumeSnapshotClass
+		var volumeSnapshotContent *snapshotv1api.VolumeSnapshotContent
+
+		// This is protective logic. The passed-in VS should be all related
+		// to this backup.
+		if volumeSnapshot.Labels[velerov1api.BackupNameLabel] != backup.Name {
+			continue
+		}
+
+		if volumeSnapshot.Spec.VolumeSnapshotClassName == nil {
+			logger.Warnf("Cannot find VolumeSnapshotClass for VolumeSnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name)
+			continue
+		}
+
+		if volumeSnapshot.Status == nil || volumeSnapshot.Status.BoundVolumeSnapshotContentName == nil {
+			logger.Warnf("Cannot fine VolumeSnapshotContent for VolumeSnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name)
+			continue
+		}
+
+		if volumeSnapshot.Spec.Source.PersistentVolumeClaimName == nil {
+			logger.Warnf("VolumeSnapshot %s/%s doesn't have a source PVC", volumeSnapshot.Namespace, volumeSnapshot.Name)
+			continue
+		}
+
+		for index := range csiVolumesnapshotClasses {
+			if *volumeSnapshot.Spec.VolumeSnapshotClassName == csiVolumesnapshotClasses[index].Name {
+				volumeSnapshotClass = &csiVolumesnapshotClasses[index]
+			}
+		}
+
+		for index := range csiVolumeSnapshotContents {
+			if *volumeSnapshot.Status.BoundVolumeSnapshotContentName == csiVolumeSnapshotContents[index].Name {
+				volumeSnapshotContent = &csiVolumeSnapshotContents[index]
+			}
+		}
+
+		if volumeSnapshotClass == nil || volumeSnapshotContent == nil {
+			logger.Warnf("fail to get VolumeSnapshotContent or VolumeSnapshotClass for VolumeSnapshot: %s/%s",
+				volumeSnapshot.Namespace, volumeSnapshot.Name)
+			continue
+		}
+
+		var operation itemoperation.BackupOperation
+		for _, op := range *backup.GetItemOperationsList() {
+			if op.Spec.ResourceIdentifier.GroupResource.String() == kuberesource.VolumeSnapshots.String() &&
+				op.Spec.ResourceIdentifier.Name == volumeSnapshot.Name &&
+				op.Spec.ResourceIdentifier.Namespace == volumeSnapshot.Namespace {
+				operation = *op
+			}
+		}
+
+		var size int64
+		if volumeSnapshot.Status.RestoreSize != nil {
+			size = volumeSnapshot.Status.RestoreSize.Value()
+		}
+		snapshotHandle := ""
+		if volumeSnapshotContent.Status.SnapshotHandle != nil {
+			snapshotHandle = *volumeSnapshotContent.Status.SnapshotHandle
+		}
+		if pvcPVInfo, ok := backup.PVMap[volumeSnapshot.Namespace+"/"+*volumeSnapshot.Spec.Source.PersistentVolumeClaimName]; ok {
+			volumeInfo := volume.VolumeInfo{
+				BackupMethod:          volume.CSISnapshot,
+				PVCName:               pvcPVInfo.PVCName,
+				PVCNamespace:          pvcPVInfo.PVCNamespace,
+				PVName:                pvcPVInfo.PV.Name,
+				Skipped:               false,
+				SnapshotDataMoved:     false,
+				PreserveLocalSnapshot: true,
+				OperationID:           operation.Spec.OperationID,
+				StartTimestamp:        &volumeSnapshot.CreationTimestamp,
+				CSISnapshotInfo: volume.CSISnapshotInfo{
+					VSCName:        *volumeSnapshot.Status.BoundVolumeSnapshotContentName,
+					Size:           size,
+					Driver:         volumeSnapshotClass.Driver,
+					SnapshotHandle: snapshotHandle,
+				},
+				PVInfo: volume.PVInfo{
+					ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy),
+					Labels:        pvcPVInfo.PV.Labels,
+				},
+			}
+
+			tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo)
+		} else {
+			logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Spec.Source.PersistentVolumeClaimName)
+			continue
+		}
+	}
+
+	return tmpVolumeInfos
+}
+
+// generateVolumeInfoFromPVB generate VolumeInfo for PVB.
+func generateVolumeInfoFromPVB(backup *pkgbackup.Request, crClient kbclient.Client, logger logrus.FieldLogger) []volume.VolumeInfo {
+	tmpVolumeInfos := make([]volume.VolumeInfo, 0)
+
+	for _, pvb := range backup.PodVolumeBackups {
+		volumeInfo := volume.VolumeInfo{
+			BackupMethod:      volume.PodVolumeBackup,
+			SnapshotDataMoved: false,
+			Skipped:           false,
+			StartTimestamp:    pvb.Status.StartTimestamp,
+			PVBInfo: volume.PodVolumeBackupInfo{
+				SnapshotHandle: pvb.Status.SnapshotID,
+				Size:           pvb.Status.Progress.TotalBytes,
+				UploaderType:   pvb.Spec.UploaderType,
+				VolumeName:     pvb.Spec.Volume,
+				PodName:        pvb.Spec.Pod.Name,
+				PodNamespace:   pvb.Spec.Pod.Namespace,
+				NodeName:       pvb.Spec.Node,
+			},
+		}
+
+		pod := new(corev1api.Pod)
+		pvcName := ""
+		err := crClient.Get(context.TODO(), kbclient.ObjectKey{Namespace: pvb.Spec.Pod.Namespace, Name: pvb.Spec.Pod.Name}, pod)
+		if err != nil {
+			logger.WithError(err).Warn("Fail to get pod for PodVolumeBackup: ", pvb.Name)
+			continue
+		}
+		for _, volume := range pod.Spec.Volumes {
+			if volume.Name == pvb.Spec.Volume && volume.PersistentVolumeClaim != nil {
+				pvcName = volume.PersistentVolumeClaim.ClaimName
+			}
+		}
+
+		if pvcName != "" {
+			if pvcPVInfo, ok := backup.PVMap[pod.Namespace+"/"+pvcName]; ok {
+				volumeInfo.PVCName = pvcPVInfo.PVCName
+				volumeInfo.PVCNamespace = pvcPVInfo.PVCNamespace
+				volumeInfo.PVName = pvcPVInfo.PV.Name
+				volumeInfo.PVInfo = volume.PVInfo{
+					ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy),
+					Labels:        pvcPVInfo.PV.Labels,
+				}
+			} else {
+				logger.Warnf("Cannot find info for PVC %s/%s", pod.Namespace, pvcName)
+				continue
+			}
+		} else {
+			logger.Debug("The PVB %s doesn't have a corresponding PVC", pvb.Name)
+		}
+
+		tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo)
+	}
+
+	return tmpVolumeInfos
+}
+
+// generateVolumeInfoFromDataUpload generate VolumeInfo for DataUpload.
+func generateVolumeInfoFromDataUpload(backup *pkgbackup.Request, crClient kbclient.Client, logger logrus.FieldLogger) []volume.VolumeInfo {
+	tmpVolumeInfos := make([]volume.VolumeInfo, 0)
+	vsClassList := new(snapshotv1api.VolumeSnapshotClassList)
+	if err := crClient.List(context.TODO(), vsClassList); err != nil {
+		logger.WithError(err).Errorf("cannot list VolumeSnapshotClass %s", err.Error())
+		return tmpVolumeInfos
+	}
+
+	for _, operation := range *backup.GetItemOperationsList() {
+		if operation.Spec.ResourceIdentifier.GroupResource.String() == kuberesource.PersistentVolumeClaims.String() {
+			var duIdentifier velero.ResourceIdentifier
+
+			for _, identifier := range operation.Spec.PostOperationItems {
+				if identifier.GroupResource.String() == "datauploads.velero.io" {
+					duIdentifier = identifier
+				}
+			}
+			if duIdentifier.Empty() {
+				logger.Warnf("cannot find DataUpload for PVC %s/%s backup async operation",
+					operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name)
+				continue
+			}
+
+			dataUpload := new(velerov2alpha1.DataUpload)
+			err := crClient.Get(
+				context.TODO(),
+				kbclient.ObjectKey{
+					Namespace: duIdentifier.Namespace,
+					Name:      duIdentifier.Name},
+				dataUpload,
+			)
+			if err != nil {
+				logger.Warnf("fail to get DataUpload for operation %s: %s", operation.Spec.OperationID, err.Error())
+				continue
+			}
+
+			driverUsedByVSClass := ""
+			for index := range vsClassList.Items {
+				if vsClassList.Items[index].Name == dataUpload.Spec.CSISnapshot.SnapshotClass {
+					driverUsedByVSClass = vsClassList.Items[index].Driver
+				}
+			}
+
+			if pvcPVInfo, ok := backup.PVMap[operation.Spec.ResourceIdentifier.Namespace+"/"+operation.Spec.ResourceIdentifier.Name]; ok {
+				volumeInfo := volume.VolumeInfo{
+					BackupMethod:      volume.CSISnapshot,
+					PVCName:           pvcPVInfo.PVCName,
+					PVCNamespace:      pvcPVInfo.PVCNamespace,
+					PVName:            pvcPVInfo.PV.Name,
+					SnapshotDataMoved: true,
+					Skipped:           false,
+					OperationID:       operation.Spec.OperationID,
+					StartTimestamp:    operation.Status.Created,
+					CSISnapshotInfo: volume.CSISnapshotInfo{
+						Driver: driverUsedByVSClass,
+					},
+					SnapshotDataMovementInfo: volume.SnapshotDataMovementInfo{
+						DataMover:    dataUpload.Spec.DataMover,
+						UploaderType: "kopia",
+					},
+					PVInfo: volume.PVInfo{
+						ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy),
+						Labels:        pvcPVInfo.PV.Labels,
+					},
+				}
+
+				tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo)
+			} else {
+				logger.Warnf("Cannot find info for PVC %s/%s", operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name)
+				continue
+			}
+		}
+	}
+
+	return tmpVolumeInfos
+}

pkg/controller/backup_controller_test.go

@@ -29,15 +29,16 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
-	snapshotfake "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake"
-	snapshotinformers "github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	corev1api "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/utils/clock"
@@ -45,11 +46,14 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/vmware-tanzu/velero/pkg/backup"
 	kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
+	"github.com/vmware-tanzu/velero/pkg/volume"
 
 	fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
 	pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	"github.com/vmware-tanzu/velero/pkg/discovery"
@@ -61,6 +65,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt"
 	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks"
+	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
@@ -1062,12 +1067,11 @@ func TestProcessBackupCompletions(t *testing.T) {
 					CSIVolumeSnapshotsCompleted: 0,
 				},
 			},
-			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
+			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
 		},
 		{
-			name:   "backup with snapshot data movement set to false when CSI feature is enabled",
-			backup: defaultBackup().SnapshotMoveData(false).Result(),
-			//backup:                   defaultBackup().Result(),
+			name:                     "backup with snapshot data movement set to false when CSI feature is enabled",
+			backup:                   defaultBackup().SnapshotMoveData(false).Result(),
 			backupLocation:           defaultBackupLocation,
 			defaultVolumesToFsBackup: false,
 			expectedResult: &velerov1api.Backup{
@@ -1103,7 +1107,7 @@ func TestProcessBackupCompletions(t *testing.T) {
 					CSIVolumeSnapshotsCompleted: 0,
 				},
 			},
-			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
+			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
 		},
 		{
 			name:                     "backup with snapshot data movement not set when CSI feature is enabled",
@@ -1143,7 +1147,7 @@ func TestProcessBackupCompletions(t *testing.T) {
 					CSIVolumeSnapshotsCompleted: 0,
 				},
 			},
-			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
+			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
 		},
 		{
 			name:                     "backup with snapshot data movement set to true and defaultSnapshotMoveData set to false",
@@ -1184,7 +1188,7 @@ func TestProcessBackupCompletions(t *testing.T) {
 					CSIVolumeSnapshotsCompleted: 0,
 				},
 			},
-			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
+			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
 		},
 		{
 			name:                     "backup with snapshot data movement set to false and defaultSnapshotMoveData set to true",
@@ -1225,7 +1229,7 @@ func TestProcessBackupCompletions(t *testing.T) {
 					CSIVolumeSnapshotsCompleted: 0,
 				},
 			},
-			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
+			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
 		},
 		{
 			name:                     "backup with snapshot data movement not set and defaultSnapshotMoveData set to true",
@@ -1266,35 +1270,43 @@ func TestProcessBackupCompletions(t *testing.T) {
 					CSIVolumeSnapshotsCompleted: 0,
 				},
 			},
-			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
+			volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(),
 		},
 	}
 
+	snapshotHandle := "testSnapshotID"
+
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			formatFlag := logging.FormatText
 			var (
-				logger         = logging.DefaultLogger(logrus.DebugLevel, formatFlag)
-				pluginManager  = new(pluginmocks.Manager)
-				backupStore    = new(persistencemocks.BackupStore)
-				backupper      = new(fakeBackupper)
-				snapshotClient = snapshotfake.NewSimpleClientset()
-				sharedInformer = snapshotinformers.NewSharedInformerFactory(snapshotClient, 0)
-				snapshotLister = sharedInformer.Snapshot().V1().VolumeSnapshots().Lister()
+				logger           = logging.DefaultLogger(logrus.DebugLevel, formatFlag)
+				pluginManager    = new(pluginmocks.Manager)
+				backupStore      = new(persistencemocks.BackupStore)
+				backupper        = new(fakeBackupper)
+				fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t)
 			)
 
 			var fakeClient kbclient.Client
 			// add the test's backup storage location if it's different than the default
 			if test.backupLocation != nil && test.backupLocation != defaultBackupLocation {
-				fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation)
+				fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation,
+					builder.ForVolumeSnapshotClass("testClass").Driver("testDriver").Result(),
+					builder.ForVolumeSnapshotContent("testVSC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).VolumeSnapshotClassName("testClass").Status(&snapshotv1api.VolumeSnapshotContentStatus{
+						SnapshotHandle: &snapshotHandle,
+					}).Result(),
+				)
 			} else {
-				fakeClient = velerotest.NewFakeControllerRuntimeClient(t)
+				fakeClient = velerotest.NewFakeControllerRuntimeClient(t,
+					builder.ForVolumeSnapshotClass("testClass").Driver("testDriver").Result(),
+					builder.ForVolumeSnapshotContent("testVSC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).VolumeSnapshotClassName("testClass").Status(&snapshotv1api.VolumeSnapshotContentStatus{
+						SnapshotHandle: &snapshotHandle,
+					}).Result(),
+				)
 			}
 
 			if test.volumeSnapshot != nil {
-				snapshotClient.SnapshotV1().VolumeSnapshots(test.volumeSnapshot.Namespace).Create(context.Background(), test.volumeSnapshot, metav1.CreateOptions{})
-				sharedInformer.Snapshot().V1().VolumeSnapshots().Informer().GetStore().Add(test.volumeSnapshot)
-				sharedInformer.WaitForCacheSync(make(chan struct{}))
+				require.NoError(t, fakeGlobalClient.Create(context.TODO(), test.volumeSnapshot))
 			}
 
 			apiServer := velerotest.NewAPIServer(t)
@@ -1328,8 +1340,7 @@ func TestProcessBackupCompletions(t *testing.T) {
 				backupStoreGetter:        NewFakeSingleObjectBackupStoreGetter(backupStore),
 				backupper:                backupper,
 				formatFlag:               formatFlag,
-				volumeSnapshotClient:     snapshotClient,
-				volumeSnapshotLister:     snapshotLister,
+				globalCRClient:           fakeGlobalClient,
 			}
 
 			pluginManager.On("GetBackupItemActionsV2").Return(nil, nil)
@@ -1731,3 +1742,749 @@ func TestPatchResourceWorksWithStatus(t *testing.T) {
 
 	}
 }
+func TestGenerateVolumeInfoForSkippedPV(t *testing.T) {
+	tests := []struct {
+		name                string
+		skippedPVName       string
+		pvMap               map[string]backup.PvcPvInfo
+		expectedVolumeInfos []volume.VolumeInfo
+	}{
+		{
+			name:                "Cannot find info for PV",
+			skippedPVName:       "testPV",
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name:          "Normal Skipped PV info",
+			skippedPVName: "testPV",
+			pvMap: map[string]backup.PvcPvInfo{
+				"velero/testPVC": {
+					PVCName:      "testPVC",
+					PVCNamespace: "velero",
+					PV: corev1api.PersistentVolume{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:   "testPV",
+							Labels: map[string]string{"a": "b"},
+						},
+						Spec: corev1api.PersistentVolumeSpec{
+							PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete,
+						},
+					},
+				},
+				"testPV": {
+					PVCName:      "testPVC",
+					PVCNamespace: "velero",
+					PV: corev1api.PersistentVolume{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:   "testPV",
+							Labels: map[string]string{"a": "b"},
+						},
+						Spec: corev1api.PersistentVolumeSpec{
+							PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete,
+						},
+					},
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{
+				{
+					PVCName:       "testPVC",
+					PVCNamespace:  "velero",
+					PVName:        "testPV",
+					Skipped:       true,
+					SkippedReason: "CSI: skipped for PodVolumeBackup;",
+					PVInfo: volume.PVInfo{
+						ReclaimPolicy: "Delete",
+						Labels: map[string]string{
+							"a": "b",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			request := new(backup.Request)
+			request.SkippedPVTracker = backup.NewSkipPVTracker()
+			if tc.skippedPVName != "" {
+				request.SkippedPVTracker.Track(tc.skippedPVName, "CSI", "skipped for PodVolumeBackup")
+			}
+			if tc.pvMap != nil {
+				request.PVMap = tc.pvMap
+			}
+			logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON)
+
+			volumeInfos := generateVolumeInfoForSkippedPV(request, logger)
+			require.Equal(t, tc.expectedVolumeInfos, volumeInfos)
+		})
+	}
+}
+
+func TestGenerateVolumeInfoForCSIVolumeSnapshot(t *testing.T) {
+	resourceQuantity := resource.MustParse("100Gi")
+	now := metav1.Now()
+	tests := []struct {
+		name                  string
+		volumeSnapshot        snapshotv1api.VolumeSnapshot
+		volumeSnapshotContent snapshotv1api.VolumeSnapshotContent
+		volumeSnapshotClass   snapshotv1api.VolumeSnapshotClass
+		pvMap                 map[string]backup.PvcPvInfo
+		operation             *itemoperation.BackupOperation
+		expectedVolumeInfos   []volume.VolumeInfo
+	}{
+		{
+			name: "VS doesn't have VolumeSnapshotClass name",
+			volumeSnapshot: snapshotv1api.VolumeSnapshot{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "testVS",
+					Namespace: "velero",
+				},
+				Spec: snapshotv1api.VolumeSnapshotSpec{},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "VS doesn't have status",
+			volumeSnapshot: snapshotv1api.VolumeSnapshot{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "testVS",
+					Namespace: "velero",
+				},
+				Spec: snapshotv1api.VolumeSnapshotSpec{
+					VolumeSnapshotClassName: stringPtr("testClass"),
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "VS doesn't have PVC",
+			volumeSnapshot: snapshotv1api.VolumeSnapshot{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "testVS",
+					Namespace: "velero",
+				},
+				Spec: snapshotv1api.VolumeSnapshotSpec{
+					VolumeSnapshotClassName: stringPtr("testClass"),
+				},
+				Status: &snapshotv1api.VolumeSnapshotStatus{
+					BoundVolumeSnapshotContentName: stringPtr("testContent"),
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "Cannot find VSC for VS",
+			volumeSnapshot: snapshotv1api.VolumeSnapshot{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "testVS",
+					Namespace: "velero",
+				},
+				Spec: snapshotv1api.VolumeSnapshotSpec{
+					VolumeSnapshotClassName: stringPtr("testClass"),
+					Source: snapshotv1api.VolumeSnapshotSource{
+						PersistentVolumeClaimName: stringPtr("testPVC"),
+					},
+				},
+				Status: &snapshotv1api.VolumeSnapshotStatus{
+					BoundVolumeSnapshotContentName: stringPtr("testContent"),
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "Cannot find VolumeInfo for PVC",
+			volumeSnapshot: snapshotv1api.VolumeSnapshot{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "testVS",
+					Namespace: "velero",
+				},
+				Spec: snapshotv1api.VolumeSnapshotSpec{
+					VolumeSnapshotClassName: stringPtr("testClass"),
+					Source: snapshotv1api.VolumeSnapshotSource{
+						PersistentVolumeClaimName: stringPtr("testPVC"),
+					},
+				},
+				Status: &snapshotv1api.VolumeSnapshotStatus{
+					BoundVolumeSnapshotContentName: stringPtr("testContent"),
+				},
+			},
+			volumeSnapshotClass:   *builder.ForVolumeSnapshotClass("testClass").Driver("pd.csi.storage.gke.io").Result(),
+			volumeSnapshotContent: *builder.ForVolumeSnapshotContent("testContent").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: stringPtr("testSnapshotHandle")}).Result(),
+			expectedVolumeInfos:   []volume.VolumeInfo{},
+		},
+		{
+			name: "Normal VolumeSnapshot case",
+			volumeSnapshot: snapshotv1api.VolumeSnapshot{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "testVS",
+					Namespace:         "velero",
+					CreationTimestamp: now,
+				},
+				Spec: snapshotv1api.VolumeSnapshotSpec{
+					VolumeSnapshotClassName: stringPtr("testClass"),
+					Source: snapshotv1api.VolumeSnapshotSource{
+						PersistentVolumeClaimName: stringPtr("testPVC"),
+					},
+				},
+				Status: &snapshotv1api.VolumeSnapshotStatus{
+					BoundVolumeSnapshotContentName: stringPtr("testContent"),
+					RestoreSize:                    &resourceQuantity,
+				},
+			},
+			volumeSnapshotClass:   *builder.ForVolumeSnapshotClass("testClass").Driver("pd.csi.storage.gke.io").Result(),
+			volumeSnapshotContent: *builder.ForVolumeSnapshotContent("testContent").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: stringPtr("testSnapshotHandle")}).Result(),
+			pvMap: map[string]backup.PvcPvInfo{
+				"velero/testPVC": {
+					PVCName:      "testPVC",
+					PVCNamespace: "velero",
+					PV: corev1api.PersistentVolume{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:   "testPV",
+							Labels: map[string]string{"a": "b"},
+						},
+						Spec: corev1api.PersistentVolumeSpec{
+							PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete,
+						},
+					},
+				},
+			},
+			operation: &itemoperation.BackupOperation{
+				Spec: itemoperation.BackupOperationSpec{
+					OperationID: "testID",
+					ResourceIdentifier: velero.ResourceIdentifier{
+						GroupResource: schema.GroupResource{
+							Group:    "snapshot.storage.k8s.io",
+							Resource: "volumesnapshots",
+						},
+						Namespace: "velero",
+						Name:      "testVS",
+					},
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{
+				{
+					PVCName:               "testPVC",
+					PVCNamespace:          "velero",
+					PVName:                "testPV",
+					BackupMethod:          volume.CSISnapshot,
+					OperationID:           "testID",
+					StartTimestamp:        &now,
+					PreserveLocalSnapshot: true,
+					CSISnapshotInfo: volume.CSISnapshotInfo{
+						Driver:         "pd.csi.storage.gke.io",
+						SnapshotHandle: "testSnapshotHandle",
+						Size:           107374182400,
+						VSCName:        "testContent",
+					},
+					PVInfo: volume.PVInfo{
+						ReclaimPolicy: "Delete",
+						Labels: map[string]string{
+							"a": "b",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			request := new(backup.Request)
+			request.Backup = new(velerov1api.Backup)
+			if tc.pvMap != nil {
+				request.PVMap = tc.pvMap
+			}
+			operationList := request.GetItemOperationsList()
+			if tc.operation != nil {
+				*operationList = append(*operationList, tc.operation)
+			}
+			logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON)
+
+			volumeInfos := generateVolumeInfoForCSIVolumeSnapshot(request, []snapshotv1api.VolumeSnapshot{tc.volumeSnapshot}, []snapshotv1api.VolumeSnapshotContent{tc.volumeSnapshotContent}, []snapshotv1api.VolumeSnapshotClass{tc.volumeSnapshotClass}, logger)
+			require.Equal(t, tc.expectedVolumeInfos, volumeInfos)
+		})
+	}
+
+}
+
+func TestGenerateVolumeInfoForVeleroNativeSnapshot(t *testing.T) {
+	tests := []struct {
+		name                string
+		nativeSnapshot      volume.Snapshot
+		pvMap               map[string]backup.PvcPvInfo
+		expectedVolumeInfos []volume.VolumeInfo
+	}{
+		{
+			name: "Native snapshot's IPOS pointer is nil",
+			nativeSnapshot: volume.Snapshot{
+				Spec: volume.SnapshotSpec{
+					PersistentVolumeName: "testPV",
+					VolumeIOPS:           nil,
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "Cannot find info for the PV",
+			nativeSnapshot: volume.Snapshot{
+				Spec: volume.SnapshotSpec{
+					PersistentVolumeName: "testPV",
+					VolumeIOPS:           int64Ptr(100),
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "Normal native snapshot",
+			pvMap: map[string]backup.PvcPvInfo{
+				"testPV": {
+					PVCName:      "testPVC",
+					PVCNamespace: "velero",
+					PV: corev1api.PersistentVolume{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:   "testPV",
+							Labels: map[string]string{"a": "b"},
+						},
+						Spec: corev1api.PersistentVolumeSpec{
+							PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete,
+						},
+					},
+				},
+			},
+			nativeSnapshot: volume.Snapshot{
+				Spec: volume.SnapshotSpec{
+					PersistentVolumeName: "testPV",
+					VolumeIOPS:           int64Ptr(100),
+					VolumeType:           "ssd",
+					VolumeAZ:             "us-central1-a",
+				},
+				Status: volume.SnapshotStatus{
+					ProviderSnapshotID: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c",
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{
+				{
+					PVCName:      "testPVC",
+					PVCNamespace: "velero",
+					PVName:       "testPV",
+					BackupMethod: volume.NativeSnapshot,
+					PVInfo: volume.PVInfo{
+						ReclaimPolicy: "Delete",
+						Labels: map[string]string{
+							"a": "b",
+						},
+					},
+					NativeSnapshotInfo: volume.NativeSnapshotInfo{
+						SnapshotHandle: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c",
+						VolumeType:     "ssd",
+						VolumeAZ:       "us-central1-a",
+						IOPS:           "100",
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			request := new(backup.Request)
+			request.VolumeSnapshots = append(request.VolumeSnapshots, &tc.nativeSnapshot)
+			if tc.pvMap != nil {
+				request.PVMap = tc.pvMap
+			}
+			logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON)
+
+			volumeInfos := generateVolumeInfoForVeleroNativeSnapshot(request, logger)
+			require.Equal(t, tc.expectedVolumeInfos, volumeInfos)
+		})
+	}
+}
+
+func TestGenerateVolumeInfoFromPVB(t *testing.T) {
+	tests := []struct {
+		name                string
+		pvb                 *velerov1api.PodVolumeBackup
+		pod                 *corev1api.Pod
+		pvMap               map[string]backup.PvcPvInfo
+		expectedVolumeInfos []volume.VolumeInfo
+	}{
+		{
+			name:                "cannot find PVB's pod, should fail",
+			pvb:                 builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(),
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "PVB doesn't have a related PVC",
+			pvb:  builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(),
+			pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{
+				Name: "test",
+				VolumeMounts: []corev1api.VolumeMount{
+					{
+						Name:      "testVolume",
+						MountPath: "/data",
+					},
+				},
+			}).Volumes(
+				&corev1api.Volume{
+					Name: "",
+					VolumeSource: corev1api.VolumeSource{
+						HostPath: &corev1api.HostPathVolumeSource{},
+					},
+				},
+			).Result(),
+			expectedVolumeInfos: []volume.VolumeInfo{
+				{
+					PVCName:      "",
+					PVCNamespace: "",
+					PVName:       "",
+					BackupMethod: volume.PodVolumeBackup,
+					PVBInfo: volume.PodVolumeBackupInfo{
+						PodName:      "testPod",
+						PodNamespace: "velero",
+					},
+				},
+			},
+		},
+		{
+			name: "Backup doesn't have information for PVC",
+			pvb:  builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(),
+			pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{
+				Name: "test",
+				VolumeMounts: []corev1api.VolumeMount{
+					{
+						Name:      "testVolume",
+						MountPath: "/data",
+					},
+				},
+			}).Volumes(
+				&corev1api.Volume{
+					Name: "",
+					VolumeSource: corev1api.VolumeSource{
+						PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{
+							ClaimName: "testPVC",
+						},
+					},
+				},
+			).Result(),
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "PVB's volume has a PVC",
+			pvMap: map[string]backup.PvcPvInfo{
+				"velero/testPVC": {
+					PVCName:      "testPVC",
+					PVCNamespace: "velero",
+					PV: corev1api.PersistentVolume{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:   "testPV",
+							Labels: map[string]string{"a": "b"},
+						},
+						Spec: corev1api.PersistentVolumeSpec{
+							PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete,
+						},
+					},
+				},
+			},
+			pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(),
+			pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{
+				Name: "test",
+				VolumeMounts: []corev1api.VolumeMount{
+					{
+						Name:      "testVolume",
+						MountPath: "/data",
+					},
+				},
+			}).Volumes(
+				&corev1api.Volume{
+					Name: "",
+					VolumeSource: corev1api.VolumeSource{
+						PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{
+							ClaimName: "testPVC",
+						},
+					},
+				},
+			).Result(),
+			expectedVolumeInfos: []volume.VolumeInfo{
+				{
+					PVCName:      "testPVC",
+					PVCNamespace: "velero",
+					PVName:       "testPV",
+					BackupMethod: volume.PodVolumeBackup,
+					PVBInfo: volume.PodVolumeBackupInfo{
+						PodName:      "testPod",
+						PodNamespace: "velero",
+					},
+					PVInfo: volume.PVInfo{
+						ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete),
+						Labels:        map[string]string{"a": "b"},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			crClient := velerotest.NewFakeControllerRuntimeClient(t)
+			logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON)
+			request := new(pkgbackup.Request)
+			request.PodVolumeBackups = append(request.PodVolumeBackups, tc.pvb)
+			if tc.pvMap != nil {
+				request.PVMap = tc.pvMap
+			}
+			if tc.pod != nil {
+				require.NoError(t, crClient.Create(context.TODO(), tc.pod))
+			}
+
+			volumeInfos := generateVolumeInfoFromPVB(request, crClient, logger)
+			require.Equal(t, tc.expectedVolumeInfos, volumeInfos)
+		})
+	}
+}
+
+func TestGenerateVolumeInfoFromDataUpload(t *testing.T) {
+	now := metav1.Now()
+	tests := []struct {
+		name                string
+		volumeSnapshotClass *snapshotv1api.VolumeSnapshotClass
+		dataUpload          *velerov2alpha1.DataUpload
+		operation           *itemoperation.BackupOperation
+		pvMap               map[string]backup.PvcPvInfo
+		expectedVolumeInfos []volume.VolumeInfo
+	}{
+		{
+			name: "Operation is not for PVC",
+			operation: &itemoperation.BackupOperation{
+				Spec: itemoperation.BackupOperationSpec{
+					ResourceIdentifier: velero.ResourceIdentifier{
+						GroupResource: schema.GroupResource{
+							Group:    "",
+							Resource: "configmaps",
+						},
+					},
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "Operation doesn't have DataUpload PostItemOperation",
+			operation: &itemoperation.BackupOperation{
+				Spec: itemoperation.BackupOperationSpec{
+					ResourceIdentifier: velero.ResourceIdentifier{
+						GroupResource: schema.GroupResource{
+							Group:    "",
+							Resource: "persistentvolumeclaims",
+						},
+						Namespace: "velero",
+						Name:      "testPVC",
+					},
+					PostOperationItems: []velero.ResourceIdentifier{
+						{
+							GroupResource: schema.GroupResource{
+								Group:    "",
+								Resource: "configmaps",
+							},
+						},
+					},
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "DataUpload cannot be found for operation",
+			operation: &itemoperation.BackupOperation{
+				Spec: itemoperation.BackupOperationSpec{
+					OperationID: "testOperation",
+					ResourceIdentifier: velero.ResourceIdentifier{
+						GroupResource: schema.GroupResource{
+							Group:    "",
+							Resource: "persistentvolumeclaims",
+						},
+						Namespace: "velero",
+						Name:      "testPVC",
+					},
+					PostOperationItems: []velero.ResourceIdentifier{
+						{
+							GroupResource: schema.GroupResource{
+								Group:    "velero.io",
+								Resource: "datauploads",
+							},
+							Namespace: "velero",
+							Name:      "testDU",
+						},
+					},
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{},
+		},
+		{
+			name: "VolumeSnapshotClass cannot be found for operation",
+			dataUpload: builder.ForDataUpload("velero", "testDU").DataMover("velero").CSISnapshot(&velerov2alpha1.CSISnapshotSpec{
+				VolumeSnapshot: "testVS",
+			}).SnapshotID("testSnapshotHandle").Result(),
+			operation: &itemoperation.BackupOperation{
+				Spec: itemoperation.BackupOperationSpec{
+					OperationID: "testOperation",
+					ResourceIdentifier: velero.ResourceIdentifier{
+						GroupResource: schema.GroupResource{
+							Group:    "",
+							Resource: "persistentvolumeclaims",
+						},
+						Namespace: "velero",
+						Name:      "testPVC",
+					},
+					PostOperationItems: []velero.ResourceIdentifier{
+						{
+							GroupResource: schema.GroupResource{
+								Group:    "velero.io",
+								Resource: "datauploads",
+							},
+							Namespace: "velero",
+							Name:      "testDU",
+						},
+					},
+				},
+			},
+			pvMap: map[string]backup.PvcPvInfo{
+				"velero/testPVC": {
+					PVCName:      "testPVC",
+					PVCNamespace: "velero",
+					PV: corev1api.PersistentVolume{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:   "testPV",
+							Labels: map[string]string{"a": "b"},
+						},
+						Spec: corev1api.PersistentVolumeSpec{
+							PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete,
+						},
+					},
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{
+				{
+					PVCName:           "testPVC",
+					PVCNamespace:      "velero",
+					PVName:            "testPV",
+					BackupMethod:      volume.CSISnapshot,
+					SnapshotDataMoved: true,
+					OperationID:       "testOperation",
+					SnapshotDataMovementInfo: volume.SnapshotDataMovementInfo{
+						DataMover:    "velero",
+						UploaderType: "kopia",
+					},
+					PVInfo: volume.PVInfo{
+						ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete),
+						Labels:        map[string]string{"a": "b"},
+					},
+				},
+			},
+		},
+		{
+			name: "Normal DataUpload case",
+			dataUpload: builder.ForDataUpload("velero", "testDU").DataMover("velero").CSISnapshot(&velerov2alpha1.CSISnapshotSpec{
+				VolumeSnapshot: "testVS",
+				SnapshotClass:  "testClass",
+			}).SnapshotID("testSnapshotHandle").Result(),
+			volumeSnapshotClass: builder.ForVolumeSnapshotClass("testClass").Driver("pd.csi.storage.gke.io").Result(),
+			operation: &itemoperation.BackupOperation{
+				Spec: itemoperation.BackupOperationSpec{
+					OperationID: "testOperation",
+					ResourceIdentifier: velero.ResourceIdentifier{
+						GroupResource: schema.GroupResource{
+							Group:    "",
+							Resource: "persistentvolumeclaims",
+						},
+						Namespace: "velero",
+						Name:      "testPVC",
+					},
+					PostOperationItems: []velero.ResourceIdentifier{
+						{
+							GroupResource: schema.GroupResource{
+								Group:    "velero.io",
+								Resource: "datauploads",
+							},
+							Namespace: "velero",
+							Name:      "testDU",
+						},
+					},
+				},
+				Status: itemoperation.OperationStatus{
+					Created: &now,
+				},
+			},
+			pvMap: map[string]backup.PvcPvInfo{
+				"velero/testPVC": {
+					PVCName:      "testPVC",
+					PVCNamespace: "velero",
+					PV: corev1api.PersistentVolume{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:   "testPV",
+							Labels: map[string]string{"a": "b"},
+						},
+						Spec: corev1api.PersistentVolumeSpec{
+							PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete,
+						},
+					},
+				},
+			},
+			expectedVolumeInfos: []volume.VolumeInfo{
+				{
+					PVCName:           "testPVC",
+					PVCNamespace:      "velero",
+					PVName:            "testPV",
+					BackupMethod:      volume.CSISnapshot,
+					SnapshotDataMoved: true,
+					OperationID:       "testOperation",
+					StartTimestamp:    &now,
+					CSISnapshotInfo: volume.CSISnapshotInfo{
+						Driver: "pd.csi.storage.gke.io",
+					},
+					SnapshotDataMovementInfo: volume.SnapshotDataMovementInfo{
+						DataMover:    "velero",
+						UploaderType: "kopia",
+					},
+					PVInfo: volume.PVInfo{
+						ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete),
+						Labels:        map[string]string{"a": "b"},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			request := new(backup.Request)
+			operationList := request.GetItemOperationsList()
+			if tc.operation != nil {
+				*operationList = append(*operationList, tc.operation)
+			}
+			if tc.pvMap != nil {
+				request.PVMap = tc.pvMap
+			}
+			logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON)
+
+			crClient := velerotest.NewFakeControllerRuntimeClient(t)
+			if tc.dataUpload != nil {
+				crClient.Create(context.TODO(), tc.dataUpload)
+			}
+
+			if tc.volumeSnapshotClass != nil {
+				crClient.Create(context.TODO(), tc.volumeSnapshotClass)
+			}
+
+			volumeInfos := generateVolumeInfoFromDataUpload(request, crClient, logger)
+			require.Equal(t, tc.expectedVolumeInfos, volumeInfos)
+		})
+	}
+}
+
+func int64Ptr(val int) *int64 {
+	i := int64(val)
+	return &i
+}
+
+func stringPtr(str string) *string {
+	return &str
+}

pkg/controller/backup_finalizer_controller.go

@@ -29,8 +29,6 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
-	snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
-
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/metrics"
@@ -42,21 +40,21 @@ import (
 
 // backupFinalizerReconciler reconciles a Backup object
 type backupFinalizerReconciler struct {
-	client               kbclient.Client
-	volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister
-	clock                clocks.WithTickerAndDelayedExecution
-	backupper            pkgbackup.Backupper
-	newPluginManager     func(logrus.FieldLogger) clientmgmt.Manager
-	backupTracker        BackupTracker
-	metrics              *metrics.ServerMetrics
-	backupStoreGetter    persistence.ObjectBackupStoreGetter
-	log                  logrus.FieldLogger
+	client            kbclient.Client
+	globalCRClient    kbclient.Client
+	clock             clocks.WithTickerAndDelayedExecution
+	backupper         pkgbackup.Backupper
+	newPluginManager  func(logrus.FieldLogger) clientmgmt.Manager
+	backupTracker     BackupTracker
+	metrics           *metrics.ServerMetrics
+	backupStoreGetter persistence.ObjectBackupStoreGetter
+	log               logrus.FieldLogger
 }
 
 // NewBackupFinalizerReconciler initializes and returns backupFinalizerReconciler struct.
 func NewBackupFinalizerReconciler(
 	client kbclient.Client,
-	volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister,
+	globalCRClient kbclient.Client,
 	clock clocks.WithTickerAndDelayedExecution,
 	backupper pkgbackup.Backupper,
 	newPluginManager func(logrus.FieldLogger) clientmgmt.Manager,
@@ -67,6 +65,7 @@ func NewBackupFinalizerReconciler(
 ) *backupFinalizerReconciler {
 	return &backupFinalizerReconciler{
 		client:            client,
+		globalCRClient:    globalCRClient,
 		clock:             clock,
 		backupper:         backupper,
 		newPluginManager:  newPluginManager,
@@ -191,7 +190,7 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	backup.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()}
 	recordBackupMetrics(log, backup, outBackupFile, r.metrics, true)
 
-	pkgbackup.UpdateBackupCSISnapshotsStatus(r.client, r.volumeSnapshotLister, backup, log)
+	pkgbackup.UpdateBackupCSISnapshotsStatus(r.client, r.globalCRClient, backup, log)
 	// update backup metadata in object store
 	backupJSON := new(bytes.Buffer)
 	if err := encode.To(backup, "json", backupJSON); err != nil {

pkg/controller/backup_finalizer_controller_test.go

@@ -23,7 +23,6 @@ import (
 	"testing"
 	"time"
 
-	snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -44,14 +43,13 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
-	velerotestmocks "github.com/vmware-tanzu/velero/pkg/test/mocks"
 )
 
-func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeVolumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) {
+func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeGlobalClient kbclient.Client, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) {
 	backupper := new(fakeBackupper)
 	return NewBackupFinalizerReconciler(
 		fakeClient,
-		fakeVolumeSnapshotLister,
+		fakeGlobalClient,
 		fakeClock,
 		backupper,
 		func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager },
@@ -164,9 +162,9 @@ func TestBackupFinalizerReconcile(t *testing.T) {
 
 			fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...)
 
-			fakeVolumeSnapshotLister := velerotestmocks.NewVolumeSnapshotLister(t)
+			fakeGlobalClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...)
 
-			reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeVolumeSnapshotLister, fakeClock)
+			reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeGlobalClient, fakeClock)
 			pluginManager.On("CleanupClients").Return(nil)
 			backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil)
 			backupStore.On("GetBackupContents", mock.Anything).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil)

pkg/controller/backup_operations_controller.go

@@ -19,8 +19,11 @@ package controller
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"time"
 
+	v2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2"
+
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -293,7 +296,7 @@ func getBackupItemOperationProgress(
 			if err != nil {
 				operation.Status.Phase = itemoperation.OperationPhaseFailed
 				operation.Status.Error = err.Error()
-				errs = append(errs, err.Error())
+				errs = append(errs, wrapErrMsg(err.Error(), bia))
 				changes = true
 				failedCount++
 				continue
@@ -302,7 +305,7 @@ func getBackupItemOperationProgress(
 			if err != nil {
 				operation.Status.Phase = itemoperation.OperationPhaseFailed
 				operation.Status.Error = err.Error()
-				errs = append(errs, err.Error())
+				errs = append(errs, wrapErrMsg(err.Error(), bia))
 				changes = true
 				failedCount++
 				continue
@@ -340,7 +343,7 @@ func getBackupItemOperationProgress(
 				if operationProgress.Err != "" {
 					operation.Status.Phase = itemoperation.OperationPhaseFailed
 					operation.Status.Error = operationProgress.Err
-					errs = append(errs, operationProgress.Err)
+					errs = append(errs, wrapErrMsg(operationProgress.Err, bia))
 					changes = true
 					failedCount++
 					continue
@@ -355,7 +358,7 @@ func getBackupItemOperationProgress(
 				_ = bia.Cancel(operation.Spec.OperationID, backup)
 				operation.Status.Phase = itemoperation.OperationPhaseFailed
 				operation.Status.Error = "Asynchronous action timed out"
-				errs = append(errs, operation.Status.Error)
+				errs = append(errs, wrapErrMsg(operation.Status.Error, bia))
 				changes = true
 				failedCount++
 				continue
@@ -375,3 +378,15 @@ func getBackupItemOperationProgress(
 	}
 	return inProgressOperations, changes, completedCount, failedCount, errs
 }
+
+// wrap the error message to include the BIA name
+func wrapErrMsg(errMsg string, bia v2.BackupItemAction) string {
+	plugin := "unknown"
+	if bia != nil {
+		plugin = bia.Name()
+	}
+	if len(errMsg) > 0 {
+		errMsg += ", "
+	}
+	return fmt.Sprintf("%splugin: %s", errMsg, plugin)
+}

pkg/controller/backup_operations_controller_test.go

@@ -21,6 +21,8 @@ import (
 	"testing"
 	"time"
 
+	v2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2"
+
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -286,6 +288,7 @@ func TestBackupOperationsReconcile(t *testing.T) {
 			backupStore.On("PutBackupItemOperations", mock.Anything, mock.Anything).Return(nil)
 			backupStore.On("PutBackupMetadata", mock.Anything, mock.Anything).Return(nil)
 			for _, operation := range test.backupOperations {
+				bia.On("Name").Return("test")
 				bia.On("Progress", operation.Spec.OperationID, mock.Anything).
 					Return(velero.OperationProgress{
 						Completed: test.operationComplete,
@@ -308,3 +311,40 @@ func TestBackupOperationsReconcile(t *testing.T) {
 		})
 	}
 }
+
+func TestWrapErrMsg(t *testing.T) {
+	bia2 := &biav2mocks.BackupItemAction{}
+	bia2.On("Name").Return("test-bia")
+	cases := []struct {
+		name     string
+		inputErr string
+		plugin   v2.BackupItemAction
+		expect   string
+	}{
+		{
+			name:     "empty error message",
+			inputErr: "",
+			plugin:   bia2,
+			expect:   "plugin: test-bia",
+		},
+		{
+			name:     "nil bia",
+			inputErr: "some error happened",
+			plugin:   nil,
+			expect:   "some error happened, plugin: unknown",
+		},
+		{
+			name:     "regular error and bia",
+			inputErr: "some error happened",
+			plugin:   bia2,
+			expect:   "some error happened, plugin: test-bia",
+		},
+	}
+
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			got := wrapErrMsg(test.inputErr, test.plugin)
+			assert.Equal(t, test.expect, got)
+		})
+	}
+}

pkg/controller/data_upload_controller.go

@@ -344,7 +344,7 @@ func (r *DataUploadReconciler) runCancelableDataUpload(ctx context.Context, fsBa
 	tags := map[string]string{
 		velerov1api.AsyncOperationIDLabel: du.Labels[velerov1api.AsyncOperationIDLabel],
 	}
-	if err := fsBackup.StartBackup(path, fmt.Sprintf("%s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC), "", false, tags); err != nil {
+	if err := fsBackup.StartBackup(path, fmt.Sprintf("%s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC), "", false, tags, du.Spec.UploaderConfig); err != nil {
 		return r.errorOut(ctx, du, err, "error starting data path backup", log)
 	}
 

pkg/controller/data_upload_controller_test.go

@@ -45,6 +45,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
@@ -283,7 +284,7 @@ func (f *fakeDataUploadFSBR) Init(ctx context.Context, bslName string, sourceNam
 	return nil
 }
 
-func (f *fakeDataUploadFSBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string) error {
+func (f *fakeDataUploadFSBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfigs shared.UploaderConfig) error {
 	du := f.du
 	original := f.du.DeepCopy()
 	du.Status.Phase = velerov2alpha1api.DataUploadPhaseCompleted

pkg/controller/pod_volume_backup_controller.go

@@ -178,7 +178,7 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		}
 	}
 
-	if err := fsBackup.StartBackup(path, "", parentSnapshotID, false, pvb.Spec.Tags); err != nil {
+	if err := fsBackup.StartBackup(path, "", parentSnapshotID, false, pvb.Spec.Tags, pvb.Spec.UploaderConfig); err != nil {
 		return r.errorOut(ctx, &pvb, err, "error starting data path backup", log)
 	}
 

pkg/controller/pod_volume_backup_controller_test.go

@@ -37,6 +37,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	"github.com/vmware-tanzu/velero/pkg/datapath"
@@ -103,7 +104,7 @@ func (b *fakeFSBR) Init(ctx context.Context, bslName string, sourceNamespace str
 	return nil
 }
 
-func (b *fakeFSBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string) error {
+func (b *fakeFSBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfigs shared.UploaderConfig) error {
 	pvb := b.pvb
 
 	original := b.pvb.DeepCopy()

pkg/datapath/file_system.go

@@ -24,6 +24,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 	repokey "github.com/vmware-tanzu/velero/pkg/repository/keys"
@@ -129,14 +130,14 @@ func (fs *fileSystemBR) Close(ctx context.Context) {
 	fs.log.WithField("user", fs.jobName).Info("FileSystemBR is closed")
 }
 
-func (fs *fileSystemBR) StartBackup(source AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string) error {
+func (fs *fileSystemBR) StartBackup(source AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfigs shared.UploaderConfig) error {
 	if !fs.initialized {
 		return errors.New("file system data path is not initialized")
 	}
 
 	go func() {
 		snapshotID, emptySnapshot, err := fs.uploaderProv.RunBackup(fs.ctx, source.ByPath, realSource, tags, forceFull,
-			parentSnapshot, source.VolMode, fs)
+			parentSnapshot, source.VolMode, uploaderConfigs, fs)
 
 		if err == provider.ErrorCanceled {
 			fs.callbacks.OnCancelled(context.Background(), fs.namespace, fs.jobName)

pkg/datapath/file_system_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
 	"github.com/vmware-tanzu/velero/pkg/uploader/provider"
 	providerMock "github.com/vmware-tanzu/velero/pkg/uploader/provider/mocks"
@@ -95,12 +96,12 @@ func TestAsyncBackup(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			fs := newFileSystemBR("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*fileSystemBR)
 			mockProvider := providerMock.NewProvider(t)
-			mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.err)
+			mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.err)
 			fs.uploaderProv = mockProvider
 			fs.initialized = true
 			fs.callbacks = test.callbacks
 
-			err := fs.StartBackup(AccessPoint{ByPath: test.path}, "", "", false, nil)
+			err := fs.StartBackup(AccessPoint{ByPath: test.path}, "", "", false, nil, shared.UploaderConfig{})
 			require.Equal(t, nil, err)
 
 			<-finish

pkg/datapath/mocks/types.go

@@ -11,6 +11,8 @@ import (
 	mock "github.com/stretchr/testify/mock"
 
 	repository "github.com/vmware-tanzu/velero/pkg/repository"
+
+	shared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 )
 
 // AsyncBR is an autogenerated mock type for the AsyncBR type
@@ -42,13 +44,13 @@ func (_m *AsyncBR) Init(ctx context.Context, bslName string, sourceNamespace str
 	return r0
 }
 
-// StartBackup provides a mock function with given fields: source, realSource, parentSnapshot, forceFull, tags
-func (_m *AsyncBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string) error {
-	ret := _m.Called(source, realSource, parentSnapshot, forceFull, tags)
+// StartBackup provides a mock function with given fields: source, realSource, parentSnapshot, forceFull, tags, uploaderConfig
+func (_m *AsyncBR) StartBackup(source datapath.AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfig shared.UploaderConfig) error {
+	ret := _m.Called(source, realSource, parentSnapshot, forceFull, tags, uploaderConfig)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(datapath.AccessPoint, string, string, bool, map[string]string) error); ok {
-		r0 = rf(source, realSource, parentSnapshot, forceFull, tags)
+	if rf, ok := ret.Get(0).(func(datapath.AccessPoint, string, string, bool, map[string]string, shared.UploaderConfig) error); ok {
+		r0 = rf(source, realSource, parentSnapshot, forceFull, tags, uploaderConfig)
 	} else {
 		r0 = ret.Error(0)
 	}

pkg/datapath/types.go

@@ -20,6 +20,7 @@ import (
 	"context"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 	"github.com/vmware-tanzu/velero/pkg/uploader"
 )
@@ -62,7 +63,7 @@ type AsyncBR interface {
 	Init(ctx context.Context, bslName string, sourceNamespace string, uploaderType string, repositoryType string, repoIdentifier string, repositoryEnsurer *repository.Ensurer, credentialGetter *credentials.CredentialGetter) error
 
 	// StartBackup starts an asynchronous data path instance for backup
-	StartBackup(source AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string) error
+	StartBackup(source AccessPoint, realSource string, parentSnapshot string, forceFull bool, tags map[string]string, uploaderConfig shared.UploaderConfig) error
 
 	// StartRestore starts an asynchronous data path instance for restore
 	StartRestore(snapshotID string, target AccessPoint) error

pkg/persistence/object_store_test.go

@@ -768,6 +768,13 @@ func TestGetDownloadURL(t *testing.T) {
 				velerov1api.DownloadTargetKindRestoreResourceList:   "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-resource-list.json.gz",
 			},
 		},
+		{
+			name:       "",
+			targetName: "my-backup",
+			expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{
+				velerov1api.DownloadTargetKindBackupVolumeInfos: "backups/my-backup/my-backup-volumeinfos.json.gz",
+			},
+		},
 	}
 
 	for _, test := range tests {

pkg/podvolume/backupper.go

@@ -392,6 +392,7 @@ func newPodVolumeBackup(backup *velerov1api.Backup, pod *corev1api.Pod, volume c
 			BackupStorageLocation: backup.Spec.StorageLocation,
 			RepoIdentifier:        repoIdentifier,
 			UploaderType:          uploaderType,
+			UploaderConfig:        backup.Spec.UploaderConfig,
 		},
 	}
 

pkg/uploader/kopia/snapshot.go

@@ -28,10 +28,6 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/vmware-tanzu/velero/pkg/kopia"
-	"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
-	"github.com/vmware-tanzu/velero/pkg/uploader"
-
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/fs/localfs"
 	"github.com/kopia/kopia/repo"
@@ -41,6 +37,11 @@ import (
 	"github.com/kopia/kopia/snapshot/restore"
 	"github.com/kopia/kopia/snapshot/snapshotfs"
 	"github.com/pkg/errors"
+
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
+	"github.com/vmware-tanzu/velero/pkg/kopia"
+	"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
+	"github.com/vmware-tanzu/velero/pkg/uploader"
 )
 
 // All function mainly used to make testing more convenient
@@ -104,9 +105,14 @@ func getDefaultPolicy() *policy.Policy {
 	}
 }
 
-func setupDefaultPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo) (*policy.Tree, error) {
+func setupPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, uploaderCfg shared.UploaderConfig) (*policy.Tree, error) {
 	// some internal operations from Kopia code retrieves policies from repo directly, so we need to persist the policy to repo
-	err := setPolicyFunc(ctx, rep, sourceInfo, getDefaultPolicy())
+	curPolicy := getDefaultPolicy()
+	if uploaderCfg.ParallelFilesUpload > 0 {
+		curPolicy.UploadPolicy.MaxParallelFileReads = newOptionalInt(uploaderCfg.ParallelFilesUpload)
+	}
+
+	err := setPolicyFunc(ctx, rep, sourceInfo, curPolicy)
 	if err != nil {
 		return nil, errors.Wrap(err, "error to set policy")
 	}
@@ -127,7 +133,7 @@ func setupDefaultPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceIn
 
 // Backup backup specific sourcePath and update progress
 func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string,
-	forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+	forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg shared.UploaderConfig, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 	if fsUploader == nil {
 		return nil, false, errors.New("get empty kopia uploader")
 	}
@@ -172,7 +178,7 @@ func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.Re
 	}
 
 	kopiaCtx := kopia.SetupKopiaLog(ctx, log)
-	snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, log, "Kopia Uploader")
+	snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, uploaderCfg, log, "Kopia Uploader")
 	if err != nil {
 		return nil, false, err
 	}
@@ -223,6 +229,7 @@ func SnapshotSource(
 	forceFull bool,
 	parentSnapshot string,
 	snapshotTags map[string]string,
+	uploaderCfg shared.UploaderConfig,
 	log logrus.FieldLogger,
 	description string,
 ) (string, int64, error) {
@@ -258,7 +265,7 @@ func SnapshotSource(
 		log.Infof("Using parent snapshot %s, start time %v, end time %v, description %s", previous[i].ID, previous[i].StartTime.ToTime(), previous[i].EndTime.ToTime(), previous[i].Description)
 	}
 
-	policyTree, err := setupDefaultPolicy(ctx, rep, sourceInfo)
+	policyTree, err := setupPolicy(ctx, rep, sourceInfo, uploaderCfg)
 	if err != nil {
 		return "", 0, errors.Wrapf(err, "unable to set policy for si %v", sourceInfo)
 	}

pkg/uploader/kopia/snapshot_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	repomocks "github.com/vmware-tanzu/velero/pkg/repository/mocks"
 	"github.com/vmware-tanzu/velero/pkg/uploader"
 	uploadermocks "github.com/vmware-tanzu/velero/pkg/uploader/mocks"
@@ -94,9 +95,10 @@ func TestSnapshotSource(t *testing.T) {
 	}
 
 	testCases := []struct {
-		name     string
-		args     []mockArgs
-		notError bool
+		name        string
+		args        []mockArgs
+		uploaderCfg shared.UploaderConfig
+		notError    bool
 	}{
 		{
 			name: "regular test",
@@ -150,6 +152,20 @@ func TestSnapshotSource(t *testing.T) {
 			},
 			notError: false,
 		},
+		{
+			name: "set policy with ParallelFilesUpload",
+			args: []mockArgs{
+				{methodName: "LoadSnapshot", returns: []interface{}{manifest, nil}},
+				{methodName: "SaveSnapshot", returns: []interface{}{manifest.ID, nil}},
+				{methodName: "TreeForSource", returns: []interface{}{nil, nil}},
+				{methodName: "ApplyRetentionPolicy", returns: []interface{}{nil, nil}},
+				{methodName: "SetPolicy", returns: []interface{}{nil}},
+				{methodName: "Upload", returns: []interface{}{manifest, nil}},
+				{methodName: "Flush", returns: []interface{}{nil}},
+			},
+			uploaderCfg: shared.UploaderConfig{ParallelFilesUpload: 10},
+			notError:    true,
+		},
 		{
 			name: "failed to upload snapshot",
 			args: []mockArgs{
@@ -182,7 +198,7 @@ func TestSnapshotSource(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			s := injectSnapshotFuncs()
 			MockFuncs(s, tc.args)
-			_, _, err = SnapshotSource(ctx, s.repoWriterMock, s.uploderMock, sourceInfo, rootDir, false, "/", nil, log, "TestSnapshotSource")
+			_, _, err = SnapshotSource(ctx, s.repoWriterMock, s.uploderMock, sourceInfo, rootDir, false, "/", nil, tc.uploaderCfg, log, "TestSnapshotSource")
 			if tc.notError {
 				assert.NoError(t, err)
 			} else {
@@ -630,9 +646,9 @@ func TestBackup(t *testing.T) {
 			var snapshotInfo *uploader.SnapshotInfo
 			var err error
 			if tc.isEmptyUploader {
-				snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), nil, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, tc.tags, &logrus.Logger{})
+				snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), nil, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, shared.UploaderConfig{}, tc.tags, &logrus.Logger{})
 			} else {
-				snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), s.uploderMock, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, tc.tags, &logrus.Logger{})
+				snapshotInfo, isSnapshotEmpty, err = Backup(context.Background(), s.uploderMock, s.repoWriterMock, tc.sourcePath, "", tc.forceFull, tc.parentSnapshot, tc.volMode, shared.UploaderConfig{}, tc.tags, &logrus.Logger{})
 			}
 			// Check if the returned error matches the expected error
 			if tc.expectedError != nil {

pkg/uploader/provider/kopia.go

@@ -30,6 +30,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/uploader/kopia"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	repokeys "github.com/vmware-tanzu/velero/pkg/repository/keys"
 	"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
@@ -119,6 +120,7 @@ func (kp *kopiaProvider) RunBackup(
 	forceFull bool,
 	parentSnapshot string,
 	volMode uploader.PersistentVolumeMode,
+	uploaderCfg shared.UploaderConfig,
 	updater uploader.ProgressUpdater) (string, bool, error) {
 	if updater == nil {
 		return "", false, errors.New("Need to initial backup progress updater first")
@@ -158,7 +160,7 @@ func (kp *kopiaProvider) RunBackup(
 		realSource = fmt.Sprintf("%s/%s/%s", kp.requestorType, uploader.KopiaType, realSource)
 	}
 
-	snapshotInfo, isSnapshotEmpty, err := BackupFunc(ctx, kpUploader, repoWriter, path, realSource, forceFull, parentSnapshot, volMode, tags, log)
+	snapshotInfo, isSnapshotEmpty, err := BackupFunc(ctx, kpUploader, repoWriter, path, realSource, forceFull, parentSnapshot, volMode, uploaderCfg, tags, log)
 	if err != nil {
 		if kpUploader.IsCanceled() {
 			log.Error("Kopia backup is canceled")

pkg/uploader/provider/kopia_test.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
 	"github.com/vmware-tanzu/velero/internal/credentials/mocks"
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 	udmrepo "github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
@@ -68,34 +69,34 @@ func TestRunBackup(t *testing.T) {
 
 	testCases := []struct {
 		name           string
-		hookBackupFunc func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error)
+		hookBackupFunc func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg shared.UploaderConfig, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error)
 		volMode        uploader.PersistentVolumeMode
 		notError       bool
 	}{
 		{
 			name: "success to backup",
-			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg shared.UploaderConfig, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 				return &uploader.SnapshotInfo{}, false, nil
 			},
 			notError: true,
 		},
 		{
 			name: "get error to backup",
-			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg shared.UploaderConfig, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 				return &uploader.SnapshotInfo{}, false, errors.New("failed to backup")
 			},
 			notError: false,
 		},
 		{
 			name: "got empty snapshot",
-			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg shared.UploaderConfig, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 				return nil, true, errors.New("snapshot is empty")
 			},
 			notError: false,
 		},
 		{
 			name: "success to backup block mode volume",
-			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
+			hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg shared.UploaderConfig, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) {
 				return &uploader.SnapshotInfo{}, false, nil
 			},
 			volMode:  uploader.PersistentVolumeBlock,
@@ -108,7 +109,7 @@ func TestRunBackup(t *testing.T) {
 				tc.volMode = uploader.PersistentVolumeFilesystem
 			}
 			BackupFunc = tc.hookBackupFunc
-			_, _, err := kp.RunBackup(context.Background(), "var", "", nil, false, "", tc.volMode, &updater)
+			_, _, err := kp.RunBackup(context.Background(), "var", "", nil, false, "", tc.volMode, shared.UploaderConfig{}, &updater)
 			if tc.notError {
 				assert.NoError(t, err)
 			} else {

pkg/uploader/provider/mocks/Provider.go

@@ -7,6 +7,8 @@ import (
 
 	mock "github.com/stretchr/testify/mock"
 
+	shared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
+
 	uploader "github.com/vmware-tanzu/velero/pkg/uploader"
 )
 
@@ -29,30 +31,30 @@ func (_m *Provider) Close(ctx context.Context) error {
 	return r0
 }
 
-// RunBackup provides a mock function with given fields: ctx, path, realSource, tags, forceFull, parentSnapshot, updater
-func (_m *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, updater uploader.ProgressUpdater) (string, bool, error) {
-	ret := _m.Called(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, updater)
+// RunBackup provides a mock function with given fields: ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater
+func (_m *Provider) RunBackup(ctx context.Context, path string, realSource string, tags map[string]string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, uploaderCfg shared.UploaderConfig, updater uploader.ProgressUpdater) (string, bool, error) {
+	ret := _m.Called(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 
 	var r0 string
 	var r1 bool
 	var r2 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.ProgressUpdater) (string, bool, error)); ok {
-		return rf(ctx, path, realSource, tags, forceFull, parentSnapshot, updater)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, shared.UploaderConfig, uploader.ProgressUpdater) (string, bool, error)); ok {
+		return rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.ProgressUpdater) string); ok {
-		r0 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, updater)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, shared.UploaderConfig, uploader.ProgressUpdater) string); ok {
+		r0 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 	} else {
 		r0 = ret.Get(0).(string)
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, string, map[string]string, bool, string, uploader.ProgressUpdater) bool); ok {
-		r1 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, updater)
+	if rf, ok := ret.Get(1).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, shared.UploaderConfig, uploader.ProgressUpdater) bool); ok {
+		r1 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 	} else {
 		r1 = ret.Get(1).(bool)
 	}
 
-	if rf, ok := ret.Get(2).(func(context.Context, string, string, map[string]string, bool, string, uploader.ProgressUpdater) error); ok {
-		r2 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, updater)
+	if rf, ok := ret.Get(2).(func(context.Context, string, string, map[string]string, bool, string, uploader.PersistentVolumeMode, shared.UploaderConfig, uploader.ProgressUpdater) error); ok {
+		r2 = rf(ctx, path, realSource, tags, forceFull, parentSnapshot, volMode, uploaderCfg, updater)
 	} else {
 		r2 = ret.Error(2)
 	}
@@ -60,13 +62,13 @@ func (_m *Provider) RunBackup(ctx context.Context, path string, realSource strin
 	return r0, r1, r2
 }
 
-// RunRestore provides a mock function with given fields: ctx, snapshotID, volumePath, updater
+// RunRestore provides a mock function with given fields: ctx, snapshotID, volumePath, volMode, updater
 func (_m *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, updater uploader.ProgressUpdater) error {
 	ret := _m.Called(ctx, snapshotID, volumePath, volMode, updater)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, uploader.ProgressUpdater) error); ok {
-		r0 = rf(ctx, snapshotID, volumePath, updater)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, uploader.ProgressUpdater) error); ok {
+		r0 = rf(ctx, snapshotID, volumePath, volMode, updater)
 	} else {
 		r0 = ret.Error(0)
 	}

pkg/uploader/provider/provider.go

@@ -28,6 +28,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/uploader"
 )
@@ -49,6 +50,7 @@ type Provider interface {
 		forceFull bool,
 		parentSnapshot string,
 		volMode uploader.PersistentVolumeMode,
+		uploaderCfg shared.UploaderConfig,
 		updater uploader.ProgressUpdater) (string, bool, error)
 	// RunRestore which will do restore for one specific volume with given snapshot id and return error
 	// updater is used for updating backup progress which implement by third-party

pkg/uploader/provider/restic.go

@@ -27,6 +27,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/restic"
 	"github.com/vmware-tanzu/velero/pkg/uploader"
@@ -122,6 +123,7 @@ func (rp *resticProvider) RunBackup(
 	forceFull bool,
 	parentSnapshot string,
 	volMode uploader.PersistentVolumeMode,
+	uploaderCfg shared.UploaderConfig,
 	updater uploader.ProgressUpdater) (string, bool, error) {
 	if updater == nil {
 		return "", false, errors.New("Need to initial backup progress updater first")
@@ -144,6 +146,10 @@ func (rp *resticProvider) RunBackup(
 		"parentSnapshot": parentSnapshot,
 	})
 
+	if uploaderCfg.ParallelFilesUpload > 0 {
+		log.Warnf("ParallelFilesUpload is set to %d, but restic does not support parallel file uploads. Ignoring.", uploaderCfg.ParallelFilesUpload)
+	}
+
 	backupCmd := resticBackupCMDFunc(rp.repoIdentifier, rp.credentialsFile, path, tags)
 	backupCmd.Env = rp.cmdEnv
 	backupCmd.CACertFile = rp.caCertFile

pkg/uploader/provider/restic_test.go

@@ -31,6 +31,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
+	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	"github.com/vmware-tanzu/velero/pkg/restic"
@@ -149,9 +150,9 @@ func TestResticRunBackup(t *testing.T) {
 			}
 			if !tc.nilUpdater {
 				updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()}
-				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, &updater)
+				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, shared.UploaderConfig{}, &updater)
 			} else {
-				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, nil)
+				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, shared.UploaderConfig{}, nil)
 			}
 
 			tc.rp.log.Infof("test name %v error %v", tc.name, err)

pkg/util/podvolume/pod_volume.go

@@ -46,7 +46,7 @@ func GetVolumesByPod(pod *corev1api.Pod, defaultVolumesToFsBackup bool) ([]strin
 		if pv.Secret != nil {
 			continue
 		}
-		// don't backup volumes mounting config maps. Config maps will be backed up separately.
+		// don't backup volumes mounting ConfigMaps. ConfigMaps will be backed up separately.
 		if pv.ConfigMap != nil {
 			continue
 		}

pkg/util/podvolume/pod_volume_test.go

@@ -222,7 +222,7 @@ func TestGetVolumesByPod(t *testing.T) {
 			},
 		},
 		{
-			name:                     "should exclude volumes mounting config maps",
+			name:                     "should exclude volumes mounting ConfigMaps",
 			defaultVolumesToFsBackup: true,
 			pod: &corev1api.Pod{
 				ObjectMeta: metav1.ObjectMeta{

pkg/volume/volume_info_common.go

@@ -51,6 +51,8 @@ type VolumeInfo struct {
 	SnapshotDataMoved bool `json:"snapshotDataMoved"`
 
 	// Whether the local snapshot is preserved after snapshot is moved.
+	// The local snapshot may be a result of CSI snapshot backup(no data movement)
+	// or a CSI snapshot data movement plus preserve local snapshot.
 	PreserveLocalSnapshot bool `json:"preserveLocalSnapshot"`
 
 	// Whether the Volume is skipped in this backup.
@@ -69,6 +71,7 @@ type VolumeInfo struct {
 	SnapshotDataMovementInfo SnapshotDataMovementInfo `json:"snapshotDataMovementInfo,omitempty"`
 	NativeSnapshotInfo       NativeSnapshotInfo       `json:"nativeSnapshotInfo,omitempty"`
 	PVBInfo                  PodVolumeBackupInfo      `json:"pvbInfo,omitempty"`
+	PVInfo                   PVInfo                   `json:"pvInfo,omitempty"`
 }
 
 // CSISnapshotInfo is used for displaying the CSI snapshot status
@@ -76,7 +79,7 @@ type CSISnapshotInfo struct {
 	// It's the storage provider's snapshot ID for CSI.
 	SnapshotHandle string `json:"snapshotHandle"`
 
-	// The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot.
+	// The snapshot corresponding volume size.
 	Size int64 `json:"size"`
 
 	// The name of the CSI driver.
@@ -91,7 +94,7 @@ type SnapshotDataMovementInfo struct {
 	// The data mover used by the backup. The valid values are `velero` and ``(equals to `velero`).
 	DataMover string `json:"dataMover"`
 
-	// The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover.
+	// The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`.
 	UploaderType string `json:"uploaderType"`
 
 	// The name or ID of the snapshot associated object(SAO).
@@ -111,9 +114,6 @@ type NativeSnapshotInfo struct {
 	// It's the storage provider's snapshot ID for the Velero-native snapshot.
 	SnapshotHandle string `json:"snapshotHandle"`
 
-	// The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot.
-	Size int64 `json:"size"`
-
 	// The cloud provider snapshot volume type.
 	VolumeType string `json:"volumeType"`
 
@@ -129,19 +129,32 @@ type PodVolumeBackupInfo struct {
 	// It's the file-system uploader's snapshot ID for PodVolumeBackup.
 	SnapshotHandle string `json:"snapshotHandle"`
 
-	// The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot.
+	// The snapshot corresponding volume size.
 	Size int64 `json:"size"`
 
-	// The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover.
+	// The type of the uploader that uploads the data. The valid values are `kopia` and `restic`.
 	UploaderType string `json:"uploaderType"`
 
 	// The PVC's corresponding volume name used by Pod
 	// https://github.com/kubernetes/kubernetes/blob/e4b74dd12fa8cb63c174091d5536a10b8ec19d34/pkg/apis/core/types.go#L48
 	VolumeName string `json:"volumeName"`
 
-	// The Pod name mounting this PVC. The format should be <namespace-name>/<pod-name>.
+	// The Pod name mounting this PVC.
 	PodName string `json:"podName"`
 
+	// The Pod namespace
+	PodNamespace string `json:"podNamespace"`
+
 	// The PVB-taken k8s node's name.
 	NodeName string `json:"nodeName"`
 }
+
+// PVInfo is used to store some PV information modified after creation.
+// Those information are lost after PV recreation.
+type PVInfo struct {
+	// ReclaimPolicy of PV. It could be different from the referenced StorageClass.
+	ReclaimPolicy string `json:"reclaimPolicy"`
+
+	// The PV's labels should be kept after recreation.
+	Labels map[string]string `json:"labels"`
+}

site/content/docs/main/file-system-backup.md

@@ -185,7 +185,7 @@ The following sections provide more details on the two approaches.
 
 In this approach, Velero will back up all pod volumes using FSB with the exception of:  
 
-- Volumes mounting the default service account token, Kubernetes secrets, and config maps
+- Volumes mounting the default service account token, Kubernetes Secrets, and ConfigMaps
 - Hostpath volumes
 
 It is possible to exclude volumes from being backed up using the `backup.velero.io/backup-volumes-excludes` 

site/content/docs/v1.10/file-system-backup.md

@@ -186,7 +186,7 @@ The following sections provide more details on the two approaches.
 
 In this approach, Velero will back up all pod volumes using FSB with the exception of:  
 
-- Volumes mounting the default service account token, Kubernetes secrets, and config maps
+- Volumes mounting the default service account token, Kubernetes Secrets, and ConfigMaps
 - Hostpath volumes
 
 It is possible to exclude volumes from being backed up using the `backup.velero.io/backup-volumes-excludes` 

site/content/docs/v1.11/file-system-backup.md

@@ -186,7 +186,7 @@ The following sections provide more details on the two approaches.
 
 In this approach, Velero will back up all pod volumes using FSB with the exception of:  
 
-- Volumes mounting the default service account token, Kubernetes secrets, and config maps
+- Volumes mounting the default service account token, Kubernetes Secrets, and ConfigMaps
 - Hostpath volumes
 
 It is possible to exclude volumes from being backed up using the `backup.velero.io/backup-volumes-excludes` 

site/content/docs/v1.12/custom-plugins.md

@@ -37,7 +37,7 @@ When naming your plugin, keep in mind that the full name needs to conform to the
 - have two parts, prefix + name, separated by '/'
 - none of the above parts can be empty
 - the prefix is a valid DNS subdomain name
-- a plugin with the same prefix + name cannot not already exist
+- a plugin with the same prefix + name cannot already exist
 
 ### Some examples:
 

site/content/docs/v1.12/file-system-backup.md

@@ -199,7 +199,7 @@ The following sections provide more details on the two approaches.
 
 In this approach, Velero will back up all pod volumes using FSB with the exception of:  
 
-- Volumes mounting the default service account token, Kubernetes secrets, and config maps
+- Volumes mounting the default service account token, Kubernetes Secrets, and ConfigMaps
 - Hostpath volumes
 
 It is possible to exclude volumes from being backed up using the `backup.velero.io/backup-volumes-excludes` 

site/content/docs/v1.7/restic.md

@@ -183,7 +183,7 @@ The following sections provide more details on the two approaches.
 
 In this approach, Velero will back up all pod volumes using restic with the exception of:
 
-- Volumes mounting the default service account token, kubernetes secrets, and config maps
+- Volumes mounting the default service account token, Kubernetes Secrets, and ConfigMaps
 - Hostpath volumes
 
 It is possible to exclude volumes from being backed up using the `backup.velero.io/backup-volumes-excludes` annotation on the pod.

site/content/docs/v1.9/restic.md

@@ -186,7 +186,7 @@ The following sections provide more details on the two approaches.
 
 In this approach, Velero will back up all pod volumes using Restic with the exception of:
 
-- Volumes mounting the default service account token, Kubernetes secrets, and config maps
+- Volumes mounting the default service account token, Kubernetes Secrets, and ConfigMaps
 - Hostpath volumes
 
 It is possible to exclude volumes from being backed up using the `backup.velero.io/backup-volumes-excludes` annotation on the pod.

site/content/posts/2020-09-16-Velero-1.5-For-And-By-Community.md

@@ -33,7 +33,7 @@ With the release of 1.5, Velero now has the ability to backup all pod volumes us
 
 1. Volumes mounting the default service account token
 1. Hostpath volumes
-1. Volumes mounting Kubernetes secrets and config maps.
+1. Volumes mounting Kubernetes Secrets and ConfigMaps.
 
 You can enable this feature on a per backup basis or as a default setting for all Velero backups. Read more about this feature on our [restic integration](https://velero.io/docs/v1.5/restic/) page on our documentation website.
 

test/e2e/backup/backup.go

@@ -31,15 +31,33 @@ import (
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
 
+type BackupRestoreTestConfig struct {
+	useVolumeSnapshots  bool
+	kibishiiPatchSubDir string
+	isRetainPVTest      bool
+}
+
 func BackupRestoreWithSnapshots() {
-	BackupRestoreTest(true)
+	config := BackupRestoreTestConfig{true, "", false}
+	BackupRestoreTest(config)
 }
 
 func BackupRestoreWithRestic() {
-	BackupRestoreTest(false)
+	config := BackupRestoreTestConfig{false, "", false}
+	BackupRestoreTest(config)
 }
 
-func BackupRestoreTest(useVolumeSnapshots bool) {
+func BackupRestoreRetainedPVWithSnapshots() {
+	config := BackupRestoreTestConfig{true, "overlays/sc-reclaim-policy/", true}
+	BackupRestoreTest(config)
+}
+
+func BackupRestoreRetainedPVWithRestic() {
+	config := BackupRestoreTestConfig{false, "overlays/sc-reclaim-policy/", true}
+	BackupRestoreTest(config)
+}
+
+func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 
 	var (
 		backupName, restoreName, kibishiiNamespace string
@@ -48,25 +66,34 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 		veleroCfg                                  VeleroConfig
 	)
 	provideSnapshotVolumesParmInBackup = false
+	useVolumeSnapshots := backupRestoreTestConfig.useVolumeSnapshots
 
 	BeforeEach(func() {
 		veleroCfg = VeleroCfg
+
+		veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir
 		veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 		veleroCfg.UseNodeAgent = !useVolumeSnapshots
 		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
 			Skip("Volume snapshots not supported on kind")
 		}
+
 		var err error
 		flag.Parse()
 		UUIDgen, err = uuid.NewRandom()
 		kibishiiNamespace = "k-" + UUIDgen.String()
 		Expect(err).To(Succeed())
+		DeleteStorageClass(context.Background(), *veleroCfg.ClientToInstallVelero, KibishiiStorageClassName)
 	})
 
 	AfterEach(func() {
 		if !veleroCfg.Debug {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
+				if backupRestoreTestConfig.isRetainPVTest {
+					CleanAllRetainedPV(context.Background(), *veleroCfg.ClientToInstallVelero)
+				}
+				DeleteStorageClass(context.Background(), *veleroCfg.ClientToInstallVelero, KibishiiStorageClassName)
 			})
 			if veleroCfg.InstallVelero {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
@@ -106,6 +133,9 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 		})
 
 		It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
+			if backupRestoreTestConfig.isRetainPVTest {
+				Skip("It's tested by 1st test case")
+			}
 			if veleroCfg.AdditionalBSLProvider == "" {
 				Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
 			}

test/e2e/basic/namespace-mapping.go

@@ -102,7 +102,7 @@ func (n *NamespaceMapping) Verify() error {
 		n.kibishiiData.Levels = len(*n.NSIncluded) + index
 		By(fmt.Sprintf("Verify workload %s after restore ", ns), func() {
 			Expect(KibishiiVerifyAfterRestore(n.Client, ns,
-				n.Ctx, n.kibishiiData)).To(Succeed(), "Fail to verify workload after restore")
+				n.Ctx, n.kibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
 		})
 	}
 	for _, ns := range *n.NSIncluded {

test/e2e/e2e_suite_test.go

@@ -102,6 +102,10 @@ var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provi
 
 var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
 
+var _ = Describe("[Basic][Snapshot][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithSnapshots)
+
+var _ = Describe("[Basic][Restic][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithRestic)
+
 var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest)
 
 var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSBackupRestore)

test/e2e/migration/migration.go

@@ -273,15 +273,16 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 			}
 
 			By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
+				//Ensure workload of "migrationNamespace" existed in cluster-A
 				ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace)
 				Expect(ns.Name).To(Equal(migrationNamespace))
-				Expect(err).NotTo(HaveOccurred())
+				Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in cluster-B err: %v", err))
 
+				//Ensure cluster-B is the target cluster
 				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
 				_, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace)
 				Expect(err).To(HaveOccurred())
 				strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")
-
 				fmt.Println(err)
 
 				veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
@@ -335,7 +336,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 
 			By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() {
 				Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace,
-					oneHourTimeout, &KibishiiData)).To(Succeed(), "Fail to verify workload after restore")
+					oneHourTimeout, &KibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
 			})
 
 			// TODO: delete backup created by case self, not all

test/e2e/pv-backup/pv-backup-filter.go

@@ -180,7 +180,7 @@ func fileContent(namespace, podName, volume string) string {
 }
 
 func fileExist(ctx context.Context, namespace, podName, volume string) error {
-	c, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
+	c, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s ",
 			FILE_NAME, volume, podName, namespace))
@@ -195,7 +195,7 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error {
 	}
 }
 func fileNotExist(ctx context.Context, namespace, podName, volume string) error {
-	_, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
+	_, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return nil
 	} else {

test/e2e/resourcepolicies/resource_policies.go

@@ -24,7 +24,6 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -164,7 +163,7 @@ func (r *ResourcePoliciesCase) Verify() error {
 					if vol.Name != volName {
 						continue
 					}
-					content, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName)
+					content, _, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName)
 					if i%2 == 0 {
 						Expect(err).To(HaveOccurred(), "Expected file not found") // File should not exist
 					} else {

test/e2e/upgrade/upgrade.go

@@ -29,7 +29,6 @@ import (
 	. "github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/util/kibishii"
-
 	. "github.com/vmware-tanzu/velero/test/util/providers"
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
@@ -256,7 +255,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 
 			By(fmt.Sprintf("Verify workload %s after restore ", upgradeNamespace), func() {
 				Expect(KibishiiVerifyAfterRestore(*veleroCfg.ClientToInstallVelero, upgradeNamespace,
-					oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore")
+					oneHourTimeout, DefaultKibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
 			})
 		})
 	})

test/util/csi/common.go

@@ -21,14 +21,12 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 )
 
@@ -128,6 +126,7 @@ func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, err
 	}
 	return snapshotHandleList, nil
 }
+
 func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
 	pvcList, err := GetPvcByPVCName(context.Background(), namespace, podName)
 	if err != nil {

test/util/k8s/common.go

@@ -104,7 +104,6 @@ func GetPvcByPVCName(ctx context.Context, namespace, pvcName string) ([]string,
 		Args: []string{"{print $1}"},
 	}
 	cmds = append(cmds, cmd)
-
 	return common.GetListByCmdPipes(ctx, cmds)
 }
 
@@ -279,15 +278,30 @@ func CreateFileToPod(ctx context.Context, namespace, podName, containerName, vol
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
 	return cmd.Run()
 }
-func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, error) {
+func FileExistInPV(ctx context.Context, namespace, podName, containerName, volume, filename string) (bool, error) {
+	stdout, stderr, err := ReadFileFromPodVolume(ctx, namespace, podName, containerName, volume, filename)
+
+	output := fmt.Sprintf("%s:%s", stdout, stderr)
+	if strings.Contains(output, fmt.Sprintf("/%s/%s: No such file or directory", volume, filename)) {
+		return false, nil
+	} else {
+		if err == nil {
+			return true, nil
+		} else {
+			return false, errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s",
+				filename, volume, podName, namespace))
+		}
+	}
+}
+func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, string, error) {
 	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
 		"--", "cat", fmt.Sprintf("/%s/%s", volume, filename)}
 	cmd := exec.CommandContext(ctx, "kubectl", arg...)
 	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
 	stdout, stderr, err := veleroexec.RunCommand(cmd)
-	fmt.Print(stdout)
-	fmt.Print(stderr)
-	return stdout, err
+	fmt.Printf("stdout: %s\n", stdout)
+	fmt.Printf("stderr: %s\n", stderr)
+	return stdout, stderr, err
 }
 
 func RunCommand(cmdName string, arg []string) string {

test/util/k8s/persistentvolumes.go

@@ -22,10 +22,9 @@ import (
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/client-go/util/retry"
-
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
 )
 
 func CreatePersistentVolume(client TestClient, name string) (*corev1.PersistentVolume, error) {
@@ -93,3 +92,16 @@ func ClearClaimRefForFailedPVs(ctx context.Context, client TestClient) error {
 
 	return nil
 }
+
+func GetAllPVNames(ctx context.Context, client TestClient) ([]string, error) {
+	var pvNameList []string
+	pvList, err := client.ClientGo.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to List PV")
+	}
+
+	for _, pvName := range pvList.Items {
+		pvNameList = append(pvNameList, pvName.Name)
+	}
+	return pvNameList, nil
+}

test/util/kibishii/kibishii_utils.go

@@ -51,6 +51,7 @@ type KibishiiData struct {
 var DefaultKibishiiWorkerCounts = 2
 var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, DefaultKibishiiWorkerCounts}
 
+var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"}
 var KibishiiPVCNameList = []string{"kibishii-data-kibishii-deployment-0", "kibishii-data-kibishii-deployment-1"}
 var KibishiiStorageClassName = "kibishii-storage-class"
 
@@ -107,6 +108,8 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 	}
 
 	fmt.Printf("VeleroBackupNamespace done %s\n", time.Now().Format("2006-01-02 15:04:05"))
+
+	// Checkpoint for a successful backup
 	if useVolumeSnapshots {
 		if providerName == "vsphere" {
 			// Wait for uploads started by the Velero Plugin for vSphere to complete
@@ -165,11 +168,49 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 		}
 	}
 
+	// Modify PV data right after backup. If PV's reclaim policy is retain, PV will be restored with the origin resource config
+	fileName := "file-" + kibishiiNamespace
+	fileBaseContent := fileName
+	fmt.Printf("Re-poulate volume  %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	for _, pod := range KibishiiPodNameList {
+		// To ensure Kibishii verification result is accurate
+		ClearKibishiiData(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data")
+
+		fileContent := fileBaseContent + pod
+		err := CreateFileToPod(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data",
+			fileName, fileContent)
+		if err != nil {
+			return errors.Wrapf(err, "failed to create file %s", fileName)
+		}
+	}
+	fmt.Printf("Re-poulate volume done %s\n", time.Now().Format("2006-01-02 15:04:05"))
+
+	pvList := []string{}
+	if strings.Contains(veleroCfg.KibishiiDirectory, "sc-reclaim-policy") {
+		// Get leftover PV list for PV cleanup
+		for _, pvc := range KibishiiPVCNameList {
+			pv, err := GetPvName(oneHourTimeout, client, pvc, kibishiiNamespace)
+			if err != nil {
+				errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
+			}
+			pvList = append(pvList, pv)
+		}
+	}
+
 	fmt.Printf("Simulating a disaster by removing namespace %s %s\n", kibishiiNamespace, time.Now().Format("2006-01-02 15:04:05"))
 	if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
 		return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
 	}
 
+	if strings.Contains(veleroCfg.KibishiiDirectory, "sc-reclaim-policy") {
+		// In scenario of CSI PV-retain-policy test, to restore PV of the backed up resource, we should make sure
+		// there are no PVs of the same name left, because in previous test step, PV's reclaim policy is retain,
+		// so PVs are not deleted although workload namespace is destroyed.
+		if err := DeletePVs(oneHourTimeout, *veleroCfg.ClientToInstallVelero, pvList); err != nil {
+			return errors.Wrapf(err, "failed to delete PVs %v", pvList)
+		}
+	}
+
 	// the snapshots of AWS may be still in pending status when do the restore, wait for a while
 	// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
 	// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
@@ -191,10 +232,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 			return errors.New(fmt.Sprintf("PVR count %d is not as expected %d", len(pvrs), pvCount))
 		}
 	}
+
 	fmt.Printf("KibishiiVerifyAfterRestore %s\n", time.Now().Format("2006-01-02 15:04:05"))
-	if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData); err != nil {
+	if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData, fileName); err != nil {
 		return errors.Wrapf(err, "Error verifying kibishii after restore")
 	}
+
 	fmt.Printf("kibishii test completed successfully %s\n", time.Now().Format("2006-01-02 15:04:05"))
 	return nil
 }
@@ -309,6 +352,15 @@ func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespa
 	return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
 }
 
+func KibishiiGenerateData(oneHourTimeout context.Context, kibishiiNamespace string, kibishiiData *KibishiiData) error {
+	fmt.Printf("generateData %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	if err := generateData(oneHourTimeout, kibishiiNamespace, kibishiiData); err != nil {
+		return errors.Wrap(err, "Failed to generate data")
+	}
+	fmt.Printf("generateData done %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	return nil
+}
+
 func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient,
 	providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures,
 	kibishiiDirectory string, useVolumeSnapshots bool, kibishiiData *KibishiiData) error {
@@ -338,16 +390,12 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie
 	if kibishiiData == nil {
 		kibishiiData = DefaultKibishiiData
 	}
-	fmt.Printf("generateData %s\n", time.Now().Format("2006-01-02 15:04:05"))
-	if err := generateData(oneHourTimeout, kibishiiNamespace, kibishiiData); err != nil {
-		return errors.Wrap(err, "Failed to generate data")
-	}
-	fmt.Printf("generateData done %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	KibishiiGenerateData(oneHourTimeout, kibishiiNamespace, kibishiiData)
 	return nil
 }
 
 func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, oneHourTimeout context.Context,
-	kibishiiData *KibishiiData) error {
+	kibishiiData *KibishiiData, incrementalFileName string) error {
 	if kibishiiData == nil {
 		kibishiiData = DefaultKibishiiData
 	}
@@ -357,6 +405,18 @@ func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, one
 	if err := waitForKibishiiPods(oneHourTimeout, client, kibishiiNamespace); err != nil {
 		return errors.Wrapf(err, "Failed to wait for ready status of kibishii pods in %s", kibishiiNamespace)
 	}
+	if incrementalFileName != "" {
+		for _, pod := range KibishiiPodNameList {
+			exist, err := FileExistInPV(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data", incrementalFileName)
+			if err != nil {
+				return errors.Wrapf(err, fmt.Sprintf("fail to get file %s", incrementalFileName))
+			}
+
+			if exist {
+				return errors.New("Unexpected incremental data exist")
+			}
+		}
+	}
 
 	// TODO - check that namespace exists
 	fmt.Printf("running kibishii verify\n")
@@ -365,3 +425,11 @@ func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, one
 	}
 	return nil
 }
+
+func ClearKibishiiData(ctx context.Context, namespace, podName, containerName, dir string) error {
+	arg := []string{"exec", "-n", namespace, "-c", containerName, podName,
+		"--", "/bin/sh", "-c", "rm -rf /" + dir + "/*"}
+	cmd := exec.CommandContext(ctx, "kubectl", arg...)
+	fmt.Printf("Kubectl exec cmd =%v\n", cmd)
+	return cmd.Run()
+}

test/util/velero/velero_utils.go

@@ -1561,3 +1561,62 @@ func InstallTestStorageClasses(path string) error {
 	}
 	return InstallStorageClass(ctx, tmpFile.Name())
 }
+
+func GetPvName(ctx context.Context, client TestClient, pvcName, namespace string) (string, error) {
+
+	pvcList, err := GetPvcByPVCName(context.Background(), namespace, pvcName)
+	if err != nil {
+		return "", err
+	}
+
+	if len(pvcList) != 1 {
+		return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], pvcName, namespace))
+	}
+
+	pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0])
+	if err != nil {
+		return "", err
+	}
+	if len(pvList) != 1 {
+		return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], pvcName, namespace))
+	}
+
+	return pvList[0], nil
+
+}
+func DeletePVs(ctx context.Context, client TestClient, pvList []string) error {
+	for _, pv := range pvList {
+		args := []string{"delete", "pv", pv, "--timeout=0s"}
+		fmt.Println(args)
+		err := exec.CommandContext(ctx, "kubectl", args...).Run()
+		if err != nil {
+			return errors.New(fmt.Sprintf("Deleted PV  %s ", pv))
+		}
+	}
+	return nil
+}
+
+func CleanAllRetainedPV(ctx context.Context, client TestClient) {
+
+	pvNameList, err := GetAllPVNames(ctx, client)
+	if err != nil {
+		fmt.Println("fail to list PV")
+	}
+	for _, pv := range pvNameList {
+		args := []string{"patch", "pv", pv, "-p", "{\"spec\":{\"persistentVolumeReclaimPolicy\":\"Delete\"}}"}
+		fmt.Println(args)
+		cmd := exec.CommandContext(ctx, "kubectl", args...)
+		stdout, errMsg, err := veleroexec.RunCommand(cmd)
+		if err != nil {
+			fmt.Printf("fail to patch PV %s reclaim policy to delete: stdout: %s, stderr: %s", pv, stdout, errMsg)
+		}
+
+		args = []string{"delete", "pv", pv, "--timeout=60s"}
+		fmt.Println(args)
+		cmd = exec.CommandContext(ctx, "kubectl", args...)
+		stdout, errMsg, err = veleroexec.RunCommand(cmd)
+		if err != nil {
+			fmt.Printf("fail to delete PV %s reclaim policy to delete: stdout: %s, stderr: %s", pv, stdout, errMsg)
+		}
+	}
+}