Merge branch 'main' of https://github.com/qiuming-best/velero into kopia-parallelism

pkg/apis/velero/v1/backup_types.go

@@ -268,12 +268,12 @@ type ExecHook struct {
 type HookErrorMode string
 
 const (
-	// HookErrorModeContinue means that an error from a hook is acceptable, and the backup can
-	// proceed.
+	// HookErrorModeContinue means that an error from a hook is acceptable and the backup/restore can
+	// proceed with the rest of hooks' execution. This backup/restore should be in `PartiallyFailed` status.
 	HookErrorModeContinue HookErrorMode = "Continue"
 
-	// HookErrorModeFail means that an error from a hook is problematic, and the backup should be in
-	// error.
+	// HookErrorModeFail means that an error from a hook is problematic and Velero should stop executing following hooks.
+	// This backup/restore should be in `PartiallyFailed` status.
 	HookErrorModeFail HookErrorMode = "Fail"
 )
 

pkg/apis/velero/v2alpha1/data_download_types.go

@@ -131,6 +131,7 @@ type DataDownloadStatus struct {
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this DataDownload was created"
 // +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the DataDownload is processed"
 
+// DataDownload acts as the protocol between data mover plugins and data mover controller for the datamover restore operation
 type DataDownload struct {
 	metav1.TypeMeta `json:",inline"`
 

pkg/apis/velero/v2alpha1/data_upload_types.go

@@ -164,6 +164,7 @@ type DataUploadStatus struct {
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this DataUpload was created"
 // +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the DataUpload is processed"
 
+// DataUpload acts as the protocol between data mover plugins and data mover controller for the datamover backup operation
 type DataUpload struct {
 	metav1.TypeMeta `json:",inline"`
 

pkg/backup/backup_test.go

@@ -46,6 +46,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	"github.com/vmware-tanzu/velero/pkg/client"
 	"github.com/vmware-tanzu/velero/pkg/discovery"
+	"github.com/vmware-tanzu/velero/pkg/features"
 	"github.com/vmware-tanzu/velero/pkg/itemoperation"
 	"github.com/vmware-tanzu/velero/pkg/kuberesource"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
@@ -1379,6 +1380,12 @@ func TestBackupItemActionsForSkippedPV(t *testing.T) {
 			expectNotSkippedPVs: []string{"pv-1"},
 		},
 	}
+	// Enable CSI feature before running the test, because Velero will check whether
+	// CSI feature is enabled before executing CSI plugin actions.
+	features.NewFeatureFlagSet("EnableCSI")
+	defer func() {
+		features.NewFeatureFlagSet("")
+	}()
 	for _, tc := range tests {
 		t.Run(tc.name, func(tt *testing.T) {
 			var (

pkg/backup/item_backupper.go

@@ -52,6 +52,7 @@ import (
 	vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1"
 	"github.com/vmware-tanzu/velero/pkg/podvolume"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
+	csiutil "github.com/vmware-tanzu/velero/pkg/util/csi"
 	pdvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume"
 	"github.com/vmware-tanzu/velero/pkg/volume"
 )
@@ -361,6 +362,14 @@ func (ib *itemBackupper) executeActions(
 			ib.trackSkippedPV(obj, groupResource, "", "skipped due to resource policy ", log)
 			continue
 		}
+
+		// If the EnableCSI feature is not enabled, but the executing action is from CSI plugin, skip the action.
+		if csiutil.ShouldSkipAction(actionName) {
+			log.Infof("Skip action %s for resource %s:%s/%s, because the CSI feature is not enabled. Feature setting is %s.",
+				actionName, groupResource.String(), metadata.GetNamespace(), metadata.GetName(), features.Serialize())
+			continue
+		}
+
 		updatedItem, additionalItemIdentifiers, operationID, postOperationItems, err := action.Execute(obj, ib.backupRequest.Backup)
 		if err != nil {
 			return nil, itemFiles, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name)
@@ -652,6 +661,7 @@ func (ib *itemBackupper) getMatchAction(obj runtime.Unstructured, groupResource
 		}
 		return ib.backupRequest.ResPolicies.GetMatchAction(pv)
 	}
+
 	return nil, nil
 }
 

pkg/backup/request.go

@@ -20,8 +20,6 @@ import (
 	"fmt"
 	"sort"
 
-	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
-
 	"github.com/vmware-tanzu/velero/internal/hook"
 	"github.com/vmware-tanzu/velero/internal/resourcepolicies"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -51,7 +49,6 @@ type Request struct {
 	VolumeSnapshots           []*volume.Snapshot
 	PodVolumeBackups          []*velerov1api.PodVolumeBackup
 	BackedUpItems             map[itemKey]struct{}
-	CSISnapshots              []snapshotv1api.VolumeSnapshot
 	itemOperationsList        *[]*itemoperation.BackupOperation
 	ResPolicies               *resourcepolicies.Policies
 	SkippedPVTracker          *skipPVTracker

pkg/backup/snapshots.go

@@ -0,0 +1,68 @@
+package backup
+
+import (
+	"context"
+
+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
+	snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/sets"
+	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/features"
+	"github.com/vmware-tanzu/velero/pkg/label"
+	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
+)
+
+// Common function to update the status of CSI snapshots
+// returns VolumeSnapshot, VolumeSnapshotContent, VolumeSnapshotClasses referenced
+func UpdateBackupCSISnapshotsStatus(client kbclient.Client, volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, backup *velerov1api.Backup, backupLog logrus.FieldLogger) (volumeSnapshots []snapshotv1api.VolumeSnapshot, volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass) {
+	if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) {
+		backupLog.Info("backup SnapshotMoveData is set to true, skip VolumeSnapshot resource persistence.")
+	} else if features.IsEnabled(velerov1api.CSIFeatureFlag) {
+		selector := label.NewSelectorForBackup(backup.Name)
+		vscList := &snapshotv1api.VolumeSnapshotContentList{}
+
+		if volumeSnapshotLister != nil {
+			tmpVSs, err := volumeSnapshotLister.List(label.NewSelectorForBackup(backup.Name))
+			if err != nil {
+				backupLog.Error(err)
+			}
+			for _, vs := range tmpVSs {
+				volumeSnapshots = append(volumeSnapshots, *vs)
+			}
+		}
+
+		err := client.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector})
+		if err != nil {
+			backupLog.Error(err)
+		}
+		if len(vscList.Items) >= 0 {
+			volumeSnapshotContents = vscList.Items
+		}
+
+		vsClassSet := sets.NewString()
+		for index := range volumeSnapshotContents {
+			// persist the volumesnapshotclasses referenced by vsc
+			if volumeSnapshotContents[index].Spec.VolumeSnapshotClassName != nil && !vsClassSet.Has(*volumeSnapshotContents[index].Spec.VolumeSnapshotClassName) {
+				vsClass := &snapshotv1api.VolumeSnapshotClass{}
+				if err := client.Get(context.TODO(), kbclient.ObjectKey{Name: *volumeSnapshotContents[index].Spec.VolumeSnapshotClassName}, vsClass); err != nil {
+					backupLog.Error(err)
+				} else {
+					vsClassSet.Insert(*volumeSnapshotContents[index].Spec.VolumeSnapshotClassName)
+					volumeSnapshotClasses = append(volumeSnapshotClasses, *vsClass)
+				}
+			}
+		}
+		backup.Status.CSIVolumeSnapshotsAttempted = len(volumeSnapshots)
+		csiVolumeSnapshotsCompleted := 0
+		for _, vs := range volumeSnapshots {
+			if vs.Status != nil && boolptr.IsSetToTrue(vs.Status.ReadyToUse) {
+				csiVolumeSnapshotsCompleted++
+			}
+		}
+		backup.Status.CSIVolumeSnapshotsCompleted = csiVolumeSnapshotsCompleted
+	}
+	return volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses
+}

pkg/builder/backup_builder.go

@@ -305,3 +305,9 @@ func (b *BackupBuilder) ParallelFilesUpload(parallel int) *BackupBuilder {
 	b.object.Spec.UploaderConfig.ParallelFilesUpload = parallel
 	return b
 }
+
+// WithStatus sets the Backup's status.
+func (b *BackupBuilder) WithStatus(status velerov1api.BackupStatus) *BackupBuilder {
+	b.object.Status = status
+	return b
+}

pkg/builder/delete_backup_request_builder.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2023 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package builder
 
 import (

pkg/builder/item_operation_builder.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2023 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package builder
 
 import (

pkg/builder/node_builder.go

@@ -41,6 +41,11 @@ func ForNode(name string) *NodeBuilder {
 	}
 }
 
+func (b *NodeBuilder) Labels(labels map[string]string) *NodeBuilder {
+	b.object.Labels = labels
+	return b
+}
+
 // Result returns the built Node.
 func (b *NodeBuilder) Result() *corev1api.Node {
 	return b.object

pkg/builder/node_selector_builder.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2023 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package builder
 
 import corev1api "k8s.io/api/core/v1"

pkg/builder/pod_builder.go

@@ -101,3 +101,8 @@ func (b *PodBuilder) ContainerStatuses(containerStatuses ...*corev1api.Container
 	}
 	return b
 }
+
+func (b *PodBuilder) Phase(phase corev1api.PodPhase) *PodBuilder {
+	b.object.Status.Phase = phase
+	return b
+}

pkg/builder/pod_volume_restore_builder.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2023 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package builder
 
 import (

pkg/builder/volume_snapshot_builder.go

@@ -67,3 +67,8 @@ func (v *VolumeSnapshotBuilder) BoundVolumeSnapshotContentName(vscName string) *
 	v.object.Status.BoundVolumeSnapshotContentName = &vscName
 	return v
 }
+
+func (v *VolumeSnapshotBuilder) SourcePVC(name string) *VolumeSnapshotBuilder {
+	v.object.Spec.Source.PersistentVolumeClaimName = &name
+	return v
+}

pkg/client/factory.go

@@ -33,7 +33,6 @@ import (
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
-	clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
 )
 
 //go:generate mockery --name Factory
@@ -42,9 +41,6 @@ import (
 type Factory interface {
 	// BindFlags binds common flags (--kubeconfig, --namespace) to the passed-in FlagSet.
 	BindFlags(flags *pflag.FlagSet)
-	// Client returns a VeleroClient. It uses the following priority to specify the cluster
-	// configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration.
-	Client() (clientset.Interface, error)
 	// KubeClient returns a Kubernetes client. It uses the following priority to specify the cluster
 	// configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration.
 	KubeClient() (kubernetes.Interface, error)
@@ -115,19 +111,6 @@ func (f *factory) ClientConfig() (*rest.Config, error) {
 	return Config(f.kubeconfig, f.kubecontext, f.baseName, f.clientQPS, f.clientBurst)
 }
 
-func (f *factory) Client() (clientset.Interface, error) {
-	clientConfig, err := f.ClientConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	veleroClient, err := clientset.NewForConfig(clientConfig)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-	return veleroClient, nil
-}
-
 func (f *factory) KubeClient() (kubernetes.Interface, error) {
 	clientConfig, err := f.ClientConfig()
 	if err != nil {

pkg/client/factory_test.go

@@ -112,11 +112,6 @@ func TestFactory(t *testing.T) {
 			assert.Equal(t, test.burst, clientConfig.Burst)
 			strings.Contains(clientConfig.UserAgent, test.baseName)
 
-			client, _ := f.Client()
-			_, e := client.Discovery().ServerGroups()
-			assert.Contains(t, e.Error(), fmt.Sprintf("Get \"%s/api?timeout=", test.expectedHost))
-			assert.NotNil(t, client)
-
 			kubeClient, _ := f.KubeClient()
 			group := kubeClient.NodeV1().RESTClient().APIVersion().Group
 			assert.NotNil(t, kubeClient)

pkg/client/mocks/Factory.go

@@ -1,4 +1,4 @@
-// Code generated by mockery v2.30.1. DO NOT EDIT.
+// Code generated by mockery v2.20.0. DO NOT EDIT.
 
 package mocks
 
@@ -13,8 +13,6 @@ import (
 	pkgclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	rest "k8s.io/client-go/rest"
-
-	versioned "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
 )
 
 // Factory is an autogenerated mock type for the Factory type
@@ -27,32 +25,6 @@ func (_m *Factory) BindFlags(flags *pflag.FlagSet) {
 	_m.Called(flags)
 }
 
-// Client provides a mock function with given fields:
-func (_m *Factory) Client() (versioned.Interface, error) {
-	ret := _m.Called()
-
-	var r0 versioned.Interface
-	var r1 error
-	if rf, ok := ret.Get(0).(func() (versioned.Interface, error)); ok {
-		return rf()
-	}
-	if rf, ok := ret.Get(0).(func() versioned.Interface); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(versioned.Interface)
-		}
-	}
-
-	if rf, ok := ret.Get(1).(func() error); ok {
-		r1 = rf()
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}
-
 // ClientConfig provides a mock function with given fields:
 func (_m *Factory) ClientConfig() (*rest.Config, error) {
 	ret := _m.Called()
@@ -212,12 +184,13 @@ func (_m *Factory) SetClientQPS(_a0 float32) {
 	_m.Called(_a0)
 }
 
-// NewFactory creates a new instance of Factory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
-// The first argument is typically a *testing.T value.
-func NewFactory(t interface {
+type mockConstructorTestingTNewFactory interface {
 	mock.TestingT
 	Cleanup(func())
-}) *Factory {
+}
+
+// NewFactory creates a new instance of Factory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewFactory(t mockConstructorTestingTNewFactory) *Factory {
 	mock := &Factory{}
 	mock.Mock.Test(t)
 

pkg/client/retry.go

@@ -25,20 +25,14 @@ import (
 )
 
 func CreateRetryGenerateName(client kbclient.Client, ctx context.Context, obj kbclient.Object) error {
-	return CreateRetryGenerateNameWithFunc(obj, func() error {
-		return client.Create(ctx, obj, &kbclient.CreateOptions{})
-	})
-}
-
-func CreateRetryGenerateNameWithFunc(obj kbclient.Object, createFn func() error) error {
 	retryCreateFn := func() error {
 		// needed to ensure that the name from the failed create isn't left on the object between retries
 		obj.SetName("")
-		return createFn()
+		return client.Create(ctx, obj, &kbclient.CreateOptions{})
 	}
 	if obj.GetGenerateName() != "" && obj.GetName() == "" {
 		return retry.OnError(retry.DefaultRetry, apierrors.IsAlreadyExists, retryCreateFn)
 	} else {
-		return createFn()
+		return client.Create(ctx, obj, &kbclient.CreateOptions{})
 	}
 }

pkg/cmd/cli/debug/debug.go

@@ -30,9 +30,12 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/vmware-tanzu/crash-diagnostics/exec"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/tools/clientcmd"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/client"
 	"github.com/vmware-tanzu/velero/pkg/cmd"
 )
@@ -110,30 +113,32 @@ func (o *option) complete(f client.Factory, fs *pflag.FlagSet) error {
 }
 
 func (o *option) validate(f client.Factory) error {
-	kubeClient, err := f.KubeClient()
+	crClient, err := f.KubebuilderClient()
 	if err != nil {
 		return err
 	}
-	l, err := kubeClient.AppsV1().Deployments(o.namespace).List(context.TODO(), metav1.ListOptions{
-		LabelSelector: "component=velero",
+	deploymentList := new(appsv1.DeploymentList)
+	selector, err := labels.Parse("component=velero")
+	cmd.CheckError(err)
+	err = crClient.List(context.TODO(), deploymentList, &ctrlclient.ListOptions{
+		Namespace:     o.namespace,
+		LabelSelector: selector,
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to check velero deployment")
 	}
-	if len(l.Items) == 0 {
+	if len(deploymentList.Items) == 0 {
 		return fmt.Errorf("velero deployment does not exist in namespace: %s", o.namespace)
 	}
-	veleroClient, err := f.Client()
-	if err != nil {
-		return err
-	}
 	if len(o.backup) > 0 {
-		if _, err := veleroClient.VeleroV1().Backups(o.namespace).Get(context.TODO(), o.backup, metav1.GetOptions{}); err != nil {
+		backup := new(velerov1api.Backup)
+		if err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: o.namespace, Name: o.backup}, backup); err != nil {
 			return err
 		}
 	}
 	if len(o.restore) > 0 {
-		if _, err := veleroClient.VeleroV1().Restores(o.namespace).Get(context.TODO(), o.restore, metav1.GetOptions{}); err != nil {
+		restore := new(velerov1api.Restore)
+		if err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: o.namespace, Name: o.restore}, restore); err != nil {
 			return err
 		}
 	}

pkg/cmd/cli/nodeagent/server.go

@@ -19,6 +19,7 @@ package nodeagent
 import (
 	"context"
 	"fmt"
+	"math"
 	"net/http"
 	"os"
 	"strings"
@@ -32,6 +33,7 @@ import (
 	storagev1api "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -53,7 +55,9 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/cmd"
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/signals"
 	"github.com/vmware-tanzu/velero/pkg/controller"
+	"github.com/vmware-tanzu/velero/pkg/datapath"
 	"github.com/vmware-tanzu/velero/pkg/metrics"
+	"github.com/vmware-tanzu/velero/pkg/nodeagent"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
 	"github.com/vmware-tanzu/velero/pkg/util/logging"
@@ -73,6 +77,7 @@ const (
 
 	defaultResourceTimeout         = 10 * time.Minute
 	defaultDataMoverPrepareTimeout = 30 * time.Minute
+	defaultDataPathConcurrentNum   = 1
 )
 
 type nodeAgentServerConfig struct {
@@ -132,6 +137,7 @@ type nodeAgentServer struct {
 	config            nodeAgentServerConfig
 	kubeClient        kubernetes.Interface
 	csiSnapshotClient *snapshotv1client.Clientset
+	dataPathMgr       *datapath.Manager
 }
 
 func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, config nodeAgentServerConfig) (*nodeAgentServer, error) {
@@ -219,6 +225,10 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi
 	if err != nil {
 		return nil, err
 	}
+
+	dataPathConcurrentNum := s.getDataPathConcurrentNum(defaultDataPathConcurrentNum)
+	s.dataPathMgr = datapath.NewManager(dataPathConcurrentNum)
+
 	return s, nil
 }
 
@@ -263,24 +273,24 @@ func (s *nodeAgentServer) run() {
 
 	credentialGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore}
 	repoEnsurer := repository.NewEnsurer(s.mgr.GetClient(), s.logger, s.config.resourceTimeout)
-	pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), repoEnsurer,
+	pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), s.dataPathMgr, repoEnsurer,
 		credentialGetter, s.nodeName, s.mgr.GetScheme(), s.metrics, s.logger)
 
 	if err := pvbReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.Fatal(err, "unable to create controller", "controller", controller.PodVolumeBackup)
 	}
 
-	if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), repoEnsurer, credentialGetter, s.logger).SetupWithManager(s.mgr); err != nil {
+	if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.dataPathMgr, repoEnsurer, credentialGetter, s.logger).SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller")
 	}
 
-	dataUploadReconciler := controller.NewDataUploadReconciler(s.mgr.GetClient(), s.kubeClient, s.csiSnapshotClient.SnapshotV1(), repoEnsurer, clock.RealClock{}, credentialGetter, s.nodeName, s.fileSystem, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
+	dataUploadReconciler := controller.NewDataUploadReconciler(s.mgr.GetClient(), s.kubeClient, s.csiSnapshotClient.SnapshotV1(), s.dataPathMgr, repoEnsurer, clock.RealClock{}, credentialGetter, s.nodeName, s.fileSystem, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
 	s.markDataUploadsCancel(dataUploadReconciler)
 	if err = dataUploadReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the data upload controller")
 	}
 
-	dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.kubeClient, repoEnsurer, credentialGetter, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
+	dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.kubeClient, s.dataPathMgr, repoEnsurer, credentialGetter, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
 	s.markDataDownloadsCancel(dataDownloadReconciler)
 	if err = dataDownloadReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the data download controller")
@@ -478,3 +488,65 @@ func (s *nodeAgentServer) markInProgressPVRsFailed(client ctrlclient.Client) {
 		s.logger.WithField("podvolumerestore", pvr.GetName()).Warn(pvr.Status.Message)
 	}
 }
+
+var getConfigsFunc = nodeagent.GetConfigs
+
+func (s *nodeAgentServer) getDataPathConcurrentNum(defaultNum int) int {
+	configs, err := getConfigsFunc(s.ctx, s.namespace, s.kubeClient)
+	if err != nil {
+		s.logger.WithError(err).Warn("Failed to get node agent configs")
+		return defaultNum
+	}
+
+	if configs == nil || configs.DataPathConcurrency == nil {
+		s.logger.Infof("Concurrency configs are not found, use the default number %v", defaultNum)
+		return defaultNum
+	}
+
+	globalNum := configs.DataPathConcurrency.GlobalConfig
+
+	if globalNum <= 0 {
+		s.logger.Warnf("Global number %v is invalid, use the default value %v", globalNum, defaultNum)
+		globalNum = defaultNum
+	}
+
+	if len(configs.DataPathConcurrency.PerNodeConfig) == 0 {
+		return globalNum
+	}
+
+	curNode, err := s.kubeClient.CoreV1().Nodes().Get(s.ctx, s.nodeName, metav1.GetOptions{})
+	if err != nil {
+		s.logger.WithError(err).Warnf("Failed to get node info for %s, use the global number %v", s.nodeName, globalNum)
+		return globalNum
+	}
+
+	concurrentNum := math.MaxInt32
+
+	for _, rule := range configs.DataPathConcurrency.PerNodeConfig {
+		selector, err := metav1.LabelSelectorAsSelector(&rule.NodeSelector)
+		if err != nil {
+			s.logger.WithError(err).Warnf("Failed to parse rule with label selector %s, skip it", rule.NodeSelector.String())
+			continue
+		}
+
+		if rule.Number <= 0 {
+			s.logger.Warnf("Rule with label selector %s is with an invalid number %v, skip it", rule.NodeSelector.String(), rule.Number)
+			continue
+		}
+
+		if selector.Matches(labels.Set(curNode.GetLabels())) {
+			if concurrentNum > rule.Number {
+				concurrentNum = rule.Number
+			}
+		}
+	}
+
+	if concurrentNum == math.MaxInt32 {
+		s.logger.Infof("Per node number for node %s is not found, use the global number %v", s.nodeName, globalNum)
+		concurrentNum = globalNum
+	} else {
+		s.logger.Infof("Use the per node number %v over global number %v for node %s", concurrentNum, globalNum, s.nodeName)
+	}
+
+	return concurrentNum
+}

pkg/cmd/cli/nodeagent/server_test.go

@@ -17,16 +17,22 @@ package nodeagent
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
 
 	"github.com/vmware-tanzu/velero/pkg/builder"
+	"github.com/vmware-tanzu/velero/pkg/nodeagent"
 	testutil "github.com/vmware-tanzu/velero/pkg/test"
 )
 
@@ -107,3 +113,259 @@ func Test_validatePodVolumesHostPath(t *testing.T) {
 		})
 	}
 }
+
+func Test_getDataPathConcurrentNum(t *testing.T) {
+	defaultNum := 100001
+	globalNum := 6
+	nodeName := "node-agent-node"
+	node1 := builder.ForNode("node-agent-node").Result()
+	node2 := builder.ForNode("node-agent-node").Labels(map[string]string{
+		"host-name": "node-1",
+		"xxxx":      "yyyyy",
+	}).Result()
+
+	invalidLabelSelector := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"inva/lid": "inva/lid",
+		},
+	}
+	validLabelSelector1 := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"host-name": "node-1",
+		},
+	}
+	validLabelSelector2 := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"xxxx": "yyyyy",
+		},
+	}
+
+	tests := []struct {
+		name          string
+		getFunc       func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error)
+		setKubeClient bool
+		kubeClientObj []runtime.Object
+		expectNum     int
+		expectLog     string
+	}{
+		{
+			name: "failed to get configs",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return nil, errors.New("fake-get-error")
+			},
+			expectLog: "Failed to get node agent configs",
+			expectNum: defaultNum,
+		},
+		{
+			name: "configs cm not found",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return nil, nil
+			},
+			expectLog: fmt.Sprintf("Concurrency configs are not found, use the default number %v", defaultNum),
+			expectNum: defaultNum,
+		},
+		{
+			name: "configs cm's data path concurrency is nil",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{}, nil
+			},
+			expectLog: fmt.Sprintf("Concurrency configs are not found, use the default number %v", defaultNum),
+			expectNum: defaultNum,
+		},
+		{
+			name: "global number is invalid",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: -1,
+					},
+				}, nil
+			},
+			expectLog: fmt.Sprintf("Global number %v is invalid, use the default value %v", -1, defaultNum),
+			expectNum: defaultNum,
+		},
+		{
+			name: "global number is valid",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+					},
+				}, nil
+			},
+			expectNum: globalNum,
+		},
+		{
+			name: "node is not found",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								Number: 100,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			expectLog:     fmt.Sprintf("Failed to get node info for %s, use the global number %v", nodeName, globalNum),
+			expectNum:     globalNum,
+		},
+		{
+			name: "failed to get selector",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: invalidLabelSelector,
+								Number:       100,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node1},
+			expectLog:     fmt.Sprintf("Failed to parse rule with label selector %s, skip it", invalidLabelSelector.String()),
+			expectNum:     globalNum,
+		},
+		{
+			name: "rule number is invalid",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       -1,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node1},
+			expectLog:     fmt.Sprintf("Rule with label selector %s is with an invalid number %v, skip it", validLabelSelector1.String(), -1),
+			expectNum:     globalNum,
+		},
+		{
+			name: "label doesn't match",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       -1,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node1},
+			expectLog:     fmt.Sprintf("Per node number for node %s is not found, use the global number %v", nodeName, globalNum),
+			expectNum:     globalNum,
+		},
+		{
+			name: "match one rule",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       66,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node2},
+			expectLog:     fmt.Sprintf("Use the per node number %v over global number %v for node %s", 66, globalNum, nodeName),
+			expectNum:     66,
+		},
+		{
+			name: "match multiple rules",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       66,
+							},
+							{
+								NodeSelector: validLabelSelector2,
+								Number:       36,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node2},
+			expectLog:     fmt.Sprintf("Use the per node number %v over global number %v for node %s", 36, globalNum, nodeName),
+			expectNum:     36,
+		},
+		{
+			name: "match multiple rules 2",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       36,
+							},
+							{
+								NodeSelector: validLabelSelector2,
+								Number:       66,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node2},
+			expectLog:     fmt.Sprintf("Use the per node number %v over global number %v for node %s", 36, globalNum, nodeName),
+			expectNum:     36,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
+
+			logBuffer := ""
+
+			s := &nodeAgentServer{
+				nodeName: nodeName,
+				logger:   testutil.NewSingleLogger(&logBuffer),
+			}
+
+			if test.setKubeClient {
+				s.kubeClient = fakeKubeClient
+			}
+
+			getConfigsFunc = test.getFunc
+
+			num := s.getDataPathConcurrentNum(defaultNum)
+			assert.Equal(t, test.expectNum, num)
+			if test.expectLog == "" {
+				assert.Equal(t, "", logBuffer)
+			} else {
+				assert.True(t, strings.Contains(logBuffer, test.expectLog))
+			}
+		})
+	}
+}

pkg/cmd/cli/repo/get.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/client"
@@ -38,19 +40,25 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command {
 			err := output.ValidateFlags(c)
 			cmd.CheckError(err)
 
-			veleroClient, err := f.Client()
+			crClient, err := f.KubebuilderClient()
 			cmd.CheckError(err)
 
 			var repos *api.BackupRepositoryList
 			if len(args) > 0 {
 				repos = new(api.BackupRepositoryList)
 				for _, name := range args {
-					repo, err := veleroClient.VeleroV1().BackupRepositories(f.Namespace()).Get(context.TODO(), name, metav1.GetOptions{})
+					repo := new(api.BackupRepository)
+					err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: f.Namespace(), Name: name}, repo)
 					cmd.CheckError(err)
 					repos.Items = append(repos.Items, *repo)
 				}
 			} else {
-				repos, err = veleroClient.VeleroV1().BackupRepositories(f.Namespace()).List(context.TODO(), listOptions)
+				selector := labels.NewSelector()
+				if listOptions.LabelSelector != "" {
+					selector, err = labels.Parse(listOptions.LabelSelector)
+					cmd.CheckError(err)
+				}
+				err = crClient.List(context.TODO(), repos, &ctrlclient.ListOptions{LabelSelector: selector})
 				cmd.CheckError(err)
 			}
 

pkg/cmd/cli/restore/logs.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/spf13/cobra"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/client"
@@ -49,13 +49,11 @@ func NewLogsCommand(f client.Factory) *cobra.Command {
 		Run: func(c *cobra.Command, args []string) {
 			restoreName := args[0]
 
-			veleroClient, err := f.Client()
-			cmd.CheckError(err)
-
 			kbClient, err := f.KubebuilderClient()
 			cmd.CheckError(err)
 
-			restore, err := veleroClient.VeleroV1().Restores(f.Namespace()).Get(context.TODO(), restoreName, metav1.GetOptions{})
+			restore := new(velerov1api.Restore)
+			err = kbClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: f.Namespace(), Name: restoreName}, restore)
 			if apierrors.IsNotFound(err) {
 				cmd.Exit("Restore %q does not exist.", restoreName)
 			} else if err != nil {

pkg/cmd/cli/schedule/create.go

@@ -115,7 +115,7 @@ func (o *CreateOptions) Complete(args []string, f client.Factory) error {
 func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 	var orders map[string]string
 
-	veleroClient, err := f.Client()
+	crClient, err := f.KubebuilderClient()
 	if err != nil {
 		return err
 	}
@@ -171,7 +171,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 		return err
 	}
 
-	_, err = veleroClient.VeleroV1().Schedules(schedule.Namespace).Create(context.TODO(), schedule, metav1.CreateOptions{})
+	err = crClient.Create(context.TODO(), schedule)
 	if err != nil {
 		return err
 	}

pkg/cmd/cli/schedule/describe.go

@@ -22,6 +22,8 @@ import (
 
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/client"
@@ -36,19 +38,25 @@ func NewDescribeCommand(f client.Factory, use string) *cobra.Command {
 		Use:   use + " [NAME1] [NAME2] [NAME...]",
 		Short: "Describe schedules",
 		Run: func(c *cobra.Command, args []string) {
-			veleroClient, err := f.Client()
+			crClient, err := f.KubebuilderClient()
 			cmd.CheckError(err)
 
 			var schedules *v1.ScheduleList
 			if len(args) > 0 {
 				schedules = new(v1.ScheduleList)
 				for _, name := range args {
-					schedule, err := veleroClient.VeleroV1().Schedules(f.Namespace()).Get(context.TODO(), name, metav1.GetOptions{})
+					schedule := new(v1.Schedule)
+					err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: f.Namespace(), Name: name}, schedule)
 					cmd.CheckError(err)
 					schedules.Items = append(schedules.Items, *schedule)
 				}
 			} else {
-				schedules, err = veleroClient.VeleroV1().Schedules(f.Namespace()).List(context.TODO(), listOptions)
+				selector := labels.NewSelector()
+				if listOptions.LabelSelector != "" {
+					selector, err = labels.Parse(listOptions.LabelSelector)
+					cmd.CheckError(err)
+				}
+				err = crClient.List(context.TODO(), schedules, &ctrlclient.ListOptions{LabelSelector: selector})
 				cmd.CheckError(err)
 			}
 

pkg/cmd/cli/schedule/get.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/client"
@@ -38,19 +40,25 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command {
 			err := output.ValidateFlags(c)
 			cmd.CheckError(err)
 
-			veleroClient, err := f.Client()
+			crClient, err := f.KubebuilderClient()
 			cmd.CheckError(err)
 
 			var schedules *api.ScheduleList
 			if len(args) > 0 {
 				schedules = new(api.ScheduleList)
 				for _, name := range args {
-					schedule, err := veleroClient.VeleroV1().Schedules(f.Namespace()).Get(context.TODO(), name, metav1.GetOptions{})
+					schedule := new(api.Schedule)
+					err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Name: name, Namespace: f.Namespace()}, schedule)
 					cmd.CheckError(err)
 					schedules.Items = append(schedules.Items, *schedule)
 				}
 			} else {
-				schedules, err = veleroClient.VeleroV1().Schedules(f.Namespace()).List(context.TODO(), listOptions)
+				selector := labels.NewSelector()
+				if listOptions.LabelSelector != "" {
+					selector, err = labels.Parse(listOptions.LabelSelector)
+					cmd.CheckError(err)
+				}
+				err := crClient.List(context.TODO(), schedules, &ctrlclient.ListOptions{LabelSelector: selector})
 				cmd.CheckError(err)
 			}
 

pkg/cmd/cli/schedule/pause.go

@@ -25,6 +25,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	kubeerrs "k8s.io/apimachinery/pkg/util/errors"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/client"
@@ -63,7 +64,7 @@ func NewPauseCommand(f client.Factory, use string) *cobra.Command {
 }
 
 func runPause(f client.Factory, o *cli.SelectOptions, paused bool) error {
-	client, err := f.Client()
+	crClient, err := f.KubebuilderClient()
 	if err != nil {
 		return err
 	}
@@ -75,7 +76,8 @@ func runPause(f client.Factory, o *cli.SelectOptions, paused bool) error {
 	switch {
 	case len(o.Names) > 0:
 		for _, name := range o.Names {
-			schedule, err := client.VeleroV1().Schedules(f.Namespace()).Get(context.TODO(), name, metav1.GetOptions{})
+			schedule := new(velerov1api.Schedule)
+			err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Name: name, Namespace: f.Namespace()}, schedule)
 			if err != nil {
 				errs = append(errs, errors.WithStack(err))
 				continue
@@ -83,11 +85,16 @@ func runPause(f client.Factory, o *cli.SelectOptions, paused bool) error {
 			schedules = append(schedules, schedule)
 		}
 	default:
-		selector := labels.Everything().String()
+		selector := labels.Everything()
 		if o.Selector.LabelSelector != nil {
-			selector = o.Selector.String()
+			convertedSelector, err := metav1.LabelSelectorAsSelector(o.Selector.LabelSelector)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			selector = convertedSelector
 		}
-		res, err := client.VeleroV1().Schedules(f.Namespace()).List(context.TODO(), metav1.ListOptions{
+		res := new(velerov1api.ScheduleList)
+		err := crClient.List(context.TODO(), res, &ctrlclient.ListOptions{
 			LabelSelector: selector,
 		})
 		if err != nil {
@@ -113,7 +120,7 @@ func runPause(f client.Factory, o *cli.SelectOptions, paused bool) error {
 			continue
 		}
 		schedule.Spec.Paused = paused
-		if _, err := client.VeleroV1().Schedules(schedule.Namespace).Update(context.TODO(), schedule, metav1.UpdateOptions{}); err != nil {
+		if err := crClient.Update(context.TODO(), schedule); err != nil {
 			return errors.Wrapf(err, "failed to update schedule %s", schedule.Name)
 		}
 		fmt.Printf("Schedule %s %s successfully\n", schedule.Name, msg)

pkg/cmd/cli/snapshotlocation/create.go

@@ -124,12 +124,12 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 		return err
 	}
 
-	client, err := f.Client()
+	client, err := f.KubebuilderClient()
 	if err != nil {
 		return err
 	}
 
-	if _, err := client.VeleroV1().VolumeSnapshotLocations(volumeSnapshotLocation.Namespace).Create(context.TODO(), volumeSnapshotLocation, metav1.CreateOptions{}); err != nil {
+	if err := client.Create(context.TODO(), volumeSnapshotLocation); err != nil {
 		return errors.WithStack(err)
 	}
 

pkg/cmd/cli/snapshotlocation/get.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/client"
@@ -36,18 +37,19 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command {
 		Run: func(c *cobra.Command, args []string) {
 			err := output.ValidateFlags(c)
 			cmd.CheckError(err)
-			veleroClient, err := f.Client()
+			client, err := f.KubebuilderClient()
 			cmd.CheckError(err)
-			var locations *api.VolumeSnapshotLocationList
+			locations := new(api.VolumeSnapshotLocationList)
+
 			if len(args) > 0 {
-				locations = new(api.VolumeSnapshotLocationList)
 				for _, name := range args {
-					location, err := veleroClient.VeleroV1().VolumeSnapshotLocations(f.Namespace()).Get(context.TODO(), name, metav1.GetOptions{})
+					location := new(api.VolumeSnapshotLocation)
+					err := client.Get(context.TODO(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: name}, location)
 					cmd.CheckError(err)
 					locations.Items = append(locations.Items, *location)
 				}
 			} else {
-				locations, err = veleroClient.VeleroV1().VolumeSnapshotLocations(f.Namespace()).List(context.TODO(), listOptions)
+				err = client.List(context.TODO(), locations, &kbclient.ListOptions{Namespace: f.Namespace()})
 				cmd.CheckError(err)
 			}
 			_, err = output.PrintWithFormat(c, locations)

pkg/cmd/server/plugin/plugin.go

@@ -152,12 +152,12 @@ func newPodVolumeRestoreItemAction(f client.Factory) plugincommon.HandlerInitial
 			return nil, err
 		}
 
-		veleroClient, err := f.Client()
+		crClient, err := f.KubebuilderClient()
 		if err != nil {
 			return nil, err
 		}
 
-		return restore.NewPodVolumeRestoreAction(logger, client.CoreV1().ConfigMaps(f.Namespace()), veleroClient.VeleroV1().PodVolumeBackups(f.Namespace())), nil
+		return restore.NewPodVolumeRestoreAction(logger, client.CoreV1().ConfigMaps(f.Namespace()), crClient), nil
 	}
 }
 

pkg/cmd/server/server.go

@@ -66,7 +66,6 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/controller"
 	velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery"
 	"github.com/vmware-tanzu/velero/pkg/features"
-	clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
 	"github.com/vmware-tanzu/velero/pkg/itemoperationmap"
 	"github.com/vmware-tanzu/velero/pkg/metrics"
 	"github.com/vmware-tanzu/velero/pkg/nodeagent"
@@ -239,7 +238,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 	command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters. Default is 10 minutes.")
 	command.Flags().IntVar(&config.maxConcurrentK8SConnections, "max-concurrent-k8s-connections", config.maxConcurrentK8SConnections, "Max concurrent connections number that Velero can create with kube-apiserver. Default is 30.")
 	command.Flags().BoolVar(&config.defaultSnapshotMoveData, "default-snapshot-move-data", config.defaultSnapshotMoveData, "Move data by default for all snapshots supporting data movement.")
-	command.Flags().BoolVar(&config.disableInformerCache, "disable-informer-cache", config.disableInformerCache, "Disable informer cache for Get calls on restore. WIth this enabled, it will speed up restore in cases where there are backup resources which already exist in the cluster, but for very large clusters this will increase velero memory usage. Default is false (don't disable).")
+	command.Flags().BoolVar(&config.disableInformerCache, "disable-informer-cache", config.disableInformerCache, "Disable informer cache for Get calls on restore. With this enabled, it will speed up restore in cases where there are backup resources which already exist in the cluster, but for very large clusters this will increase velero memory usage. Default is false (don't disable).")
 
 	return command
 }
@@ -249,12 +248,12 @@ type server struct {
 	metricsAddress        string
 	kubeClientConfig      *rest.Config
 	kubeClient            kubernetes.Interface
-	veleroClient          clientset.Interface
 	discoveryClient       discovery.DiscoveryInterface
 	discoveryHelper       velerodiscovery.Helper
 	dynamicClient         dynamic.Interface
 	csiSnapshotClient     *snapshotv1client.Clientset
 	csiSnapshotLister     snapshotv1listers.VolumeSnapshotLister
+	crClient              ctrlclient.Client
 	ctx                   context.Context
 	cancelFunc            context.CancelFunc
 	logger                logrus.FieldLogger
@@ -268,6 +267,7 @@ type server struct {
 	mgr                   manager.Manager
 	credentialFileStore   credentials.FileStore
 	credentialSecretStore credentials.SecretStore
+	featureVerifier       features.Verifier
 }
 
 func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*server, error) {
@@ -294,12 +294,12 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 		return nil, err
 	}
 
-	veleroClient, err := f.Client()
+	dynamicClient, err := f.DynamicClient()
 	if err != nil {
 		return nil, err
 	}
 
-	dynamicClient, err := f.DynamicClient()
+	crClient, err := f.KubebuilderClient()
 	if err != nil {
 		return nil, err
 	}
@@ -309,6 +309,12 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 		return nil, err
 	}
 
+	featureVerifier := features.NewVerifier(pluginRegistry)
+
+	if _, err := featureVerifier.Verify(velerov1api.CSIFeatureFlag); err != nil {
+		logger.WithError(err).Warn("CSI feature verification failed, the feature may not be ready.")
+	}
+
 	// cancelFunc is not deferred here because if it was, then ctx would immediately
 	// be canceled once this function exited, making it useless to any informers using later.
 	// That, in turn, causes the velero server to halt when the first informer tries to use it.
@@ -367,14 +373,20 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 		return nil, err
 	}
 
+	var discoveryClient *discovery.DiscoveryClient
+	if discoveryClient, err = discovery.NewDiscoveryClientForConfig(clientConfig); err != nil {
+		cancelFunc()
+		return nil, err
+	}
+
 	s := &server{
 		namespace:             f.Namespace(),
 		metricsAddress:        config.metricsAddress,
 		kubeClientConfig:      clientConfig,
 		kubeClient:            kubeClient,
-		veleroClient:          veleroClient,
-		discoveryClient:       veleroClient.Discovery(),
+		discoveryClient:       discoveryClient,
 		dynamicClient:         dynamicClient,
+		crClient:              crClient,
 		ctx:                   ctx,
 		cancelFunc:            cancelFunc,
 		logger:                logger,
@@ -384,6 +396,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 		mgr:                   mgr,
 		credentialFileStore:   credentialFileStore,
 		credentialSecretStore: credentialSecretStore,
+		featureVerifier:       featureVerifier,
 	}
 
 	// Setup CSI snapshot client and lister
@@ -528,7 +541,7 @@ High priorities:
   - PVs go before PVCs because PVCs depend on them.
   - PVCs go before pods or controllers so they can be mounted as volumes.
   - Service accounts go before secrets so service account token secrets can be filled automatically.
-  - Secrets and config maps go before pods or controllers so they can be mounted
+  - Secrets and ConfigMaps go before pods or controllers so they can be mounted
     as volumes.
   - Limit ranges go before pods or controllers so pods can use them.
   - Pods go before controllers so they can be explicitly restored and potentially
@@ -719,6 +732,11 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 		s.logger.Fatal(err, "unable to create controller", "controller", controller.BackupStorageLocation)
 	}
 
+	pvbInformer, err := s.mgr.GetCache().GetInformer(s.ctx, &velerov1api.PodVolumeBackup{})
+	if err != nil {
+		s.logger.Fatal(err, "fail to get controller-runtime informer from manager for PVB")
+	}
+
 	if _, ok := enabledRuntimeControllers[controller.Backup]; ok {
 		backupper, err := backup.NewKubernetesBackupper(
 			s.mgr.GetClient(),
@@ -728,10 +746,8 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			podvolume.NewBackupperFactory(
 				s.repoLocker,
 				s.repoEnsurer,
-				s.veleroClient,
-				s.kubeClient.CoreV1(),
-				s.kubeClient.CoreV1(),
-				s.kubeClient.CoreV1(),
+				s.crClient,
+				pvbInformer,
 				s.logger,
 			),
 			s.config.podVolumeOperationTimeout,
@@ -760,7 +776,6 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			backupStoreGetter,
 			s.config.formatFlag.Parse(),
 			s.csiSnapshotLister,
-			s.csiSnapshotClient,
 			s.credentialFileStore,
 			s.config.maxConcurrentK8SConnections,
 			s.config.defaultSnapshotMoveData,
@@ -810,10 +825,8 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			podvolume.NewBackupperFactory(
 				s.repoLocker,
 				s.repoEnsurer,
-				s.veleroClient,
-				s.kubeClient.CoreV1(),
-				s.kubeClient.CoreV1(),
-				s.kubeClient.CoreV1(),
+				s.crClient,
+				pvbInformer,
 				s.logger,
 			),
 			s.config.podVolumeOperationTimeout,
@@ -824,6 +837,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 		cmd.CheckError(err)
 		r := controller.NewBackupFinalizerReconciler(
 			s.mgr.GetClient(),
+			s.csiSnapshotLister,
 			clock.RealClock{},
 			backupper,
 			newPluginManager,
@@ -901,6 +915,11 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 		}
 	}
 
+	pvrInformer, err := s.mgr.GetCache().GetInformer(s.ctx, &velerov1api.PodVolumeRestore{})
+	if err != nil {
+		s.logger.Fatal(err, "fail to get controller-runtime informer from manager for PVR")
+	}
+
 	if _, ok := enabledRuntimeControllers[controller.Restore]; ok {
 		restorer, err := restore.NewKubernetesRestorer(
 			s.discoveryHelper,
@@ -910,10 +929,9 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			podvolume.NewRestorerFactory(
 				s.repoLocker,
 				s.repoEnsurer,
-				s.veleroClient,
-				s.kubeClient.CoreV1(),
-				s.kubeClient.CoreV1(),
 				s.kubeClient,
+				s.crClient,
+				pvrInformer,
 				s.logger,
 			),
 			s.config.podVolumeOperationTimeout,
@@ -924,6 +942,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			s.kubeClient.CoreV1().RESTClient(),
 			s.credentialFileStore,
 			s.mgr.GetClient(),
+			s.featureVerifier,
 		)
 
 		cmd.CheckError(err)
@@ -1062,13 +1081,13 @@ func markInProgressBackupsFailed(ctx context.Context, client ctrlclient.Client,
 	}
 
 	for i, backup := range backups.Items {
-		if backup.Status.Phase != velerov1api.BackupPhaseInProgress && backup.Status.Phase != velerov1api.BackupPhaseWaitingForPluginOperations {
+		if backup.Status.Phase != velerov1api.BackupPhaseInProgress {
 			log.Debugf("the status of backup %q is %q, skip", backup.GetName(), backup.Status.Phase)
 			continue
 		}
 		updated := backup.DeepCopy()
 		updated.Status.Phase = velerov1api.BackupPhaseFailed
-		updated.Status.FailureReason = fmt.Sprintf("found a backup with status %q during the server starting, mark it as %q", velerov1api.BackupPhaseInProgress, updated.Status.Phase)
+		updated.Status.FailureReason = fmt.Sprintf("found a backup with status %q during the server starting, mark it as %q", backup.Status.Phase, updated.Status.Phase)
 		updated.Status.CompletionTimestamp = &metav1.Time{Time: time.Now()}
 		if err := client.Patch(ctx, updated, ctrlclient.MergeFrom(&backups.Items[i])); err != nil {
 			log.WithError(errors.WithStack(err)).Errorf("failed to patch backup %q", backup.GetName())
@@ -1086,13 +1105,13 @@ func markInProgressRestoresFailed(ctx context.Context, client ctrlclient.Client,
 		return
 	}
 	for i, restore := range restores.Items {
-		if restore.Status.Phase != velerov1api.RestorePhaseInProgress && restore.Status.Phase != velerov1api.RestorePhaseWaitingForPluginOperations {
+		if restore.Status.Phase != velerov1api.RestorePhaseInProgress {
 			log.Debugf("the status of restore %q is %q, skip", restore.GetName(), restore.Status.Phase)
 			continue
 		}
 		updated := restore.DeepCopy()
 		updated.Status.Phase = velerov1api.RestorePhaseFailed
-		updated.Status.FailureReason = fmt.Sprintf("found a restore with status %q during the server starting, mark it as %q", velerov1api.RestorePhaseInProgress, updated.Status.Phase)
+		updated.Status.FailureReason = fmt.Sprintf("found a restore with status %q during the server starting, mark it as %q", restore.Status.Phase, updated.Status.Phase)
 		updated.Status.CompletionTimestamp = &metav1.Time{Time: time.Now()}
 		if err := client.Patch(ctx, updated, ctrlclient.MergeFrom(&restores.Items[i])); err != nil {
 			log.WithError(errors.WithStack(err)).Errorf("failed to patch restore %q", restore.GetName())
@@ -1115,7 +1134,9 @@ func markDataUploadsCancel(ctx context.Context, client ctrlclient.Client, backup
 		du := dataUploads.Items[i]
 		if du.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted ||
 			du.Status.Phase == velerov2alpha1api.DataUploadPhasePrepared ||
-			du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress {
+			du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress ||
+			du.Status.Phase == velerov2alpha1api.DataUploadPhaseNew ||
+			du.Status.Phase == "" {
 			err := controller.UpdateDataUploadWithRetry(ctx, client, types.NamespacedName{Namespace: du.Namespace, Name: du.Name}, log.WithField("dataupload", du.Name),
 				func(dataUpload *velerov2alpha1api.DataUpload) {
 					dataUpload.Spec.Cancel = true
@@ -1143,7 +1164,9 @@ func markDataDownloadsCancel(ctx context.Context, client ctrlclient.Client, rest
 		dd := dataDownloads.Items[i]
 		if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted ||
 			dd.Status.Phase == velerov2alpha1api.DataDownloadPhasePrepared ||
-			dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress {
+			dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress ||
+			dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseNew ||
+			dd.Status.Phase == "" {
 			err := controller.UpdateDataDownloadWithRetry(ctx, client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log.WithField("datadownload", dd.Name),
 				func(dataDownload *velerov2alpha1api.DataDownload) {
 					dataDownload.Spec.Cancel = true
@@ -1151,7 +1174,7 @@ func markDataDownloadsCancel(ctx context.Context, client ctrlclient.Client, rest
 				})
 
 			if err != nil {
-				log.WithError(errors.WithStack(err)).Errorf("failed to mark dataupload %q cancel", dd.GetName())
+				log.WithError(errors.WithStack(err)).Errorf("failed to mark datadownload %q cancel", dd.GetName())
 				continue
 			}
 			log.WithField("datadownload", dd.GetName()).Warn(dd.Status.Message)

pkg/controller/backup_controller.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
@@ -33,7 +34,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
-	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/utils/clock"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -111,7 +111,6 @@ func NewBackupReconciler(
 	backupStoreGetter persistence.ObjectBackupStoreGetter,
 	formatFlag logging.Format,
 	volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister,
-	volumeSnapshotClient snapshotterClientSet.Interface,
 	credentialStore credentials.FileStore,
 	maxConcurrentK8SConnections int,
 	defaultSnapshotMoveData bool,
@@ -137,7 +136,6 @@ func NewBackupReconciler(
 		backupStoreGetter:           backupStoreGetter,
 		formatFlag:                  formatFlag,
 		volumeSnapshotLister:        volumeSnapshotLister,
-		volumeSnapshotClient:        volumeSnapshotClient,
 		credentialFileStore:         credentialStore,
 		maxConcurrentK8SConnections: maxConcurrentK8SConnections,
 		defaultSnapshotMoveData:     defaultSnapshotMoveData,
@@ -476,7 +474,7 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg
 		request.Status.ValidationErrors = append(request.Status.ValidationErrors, "encountered labelSelector as well as orLabelSelectors in backup spec, only one can be specified")
 	}
 
-	if request.Spec.ResourcePolicy != nil && request.Spec.ResourcePolicy.Kind == resourcepolicies.ConfigmapRefType {
+	if request.Spec.ResourcePolicy != nil && strings.EqualFold(request.Spec.ResourcePolicy.Kind, resourcepolicies.ConfigmapRefType) {
 		policiesConfigmap := &corev1api.ConfigMap{}
 		err := b.kbClient.Get(context.Background(), kbclient.ObjectKey{Namespace: request.Namespace, Name: request.Spec.ResourcePolicy.Name}, policiesConfigmap)
 		if err != nil {
@@ -659,65 +657,15 @@ func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error {
 		fatalErrs = append(fatalErrs, err)
 	}
 
-	// Empty slices here so that they can be passed in to the persistBackup call later, regardless of whether or not CSI's enabled.
-	// This way, we only make the Lister call if the feature flag's on.
-	var volumeSnapshots []snapshotv1api.VolumeSnapshot
-	var volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent
-	var volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass
-	if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) {
-		backupLog.Info("backup SnapshotMoveData is set to true, skip VolumeSnapshot resource persistence.")
-	} else if features.IsEnabled(velerov1api.CSIFeatureFlag) {
-		selector := label.NewSelectorForBackup(backup.Name)
-		vscList := &snapshotv1api.VolumeSnapshotContentList{}
-
-		if b.volumeSnapshotLister != nil {
-			tmpVSs, err := b.volumeSnapshotLister.List(label.NewSelectorForBackup(backup.Name))
-			if err != nil {
-				backupLog.Error(err)
-			}
-			for _, vs := range tmpVSs {
-				volumeSnapshots = append(volumeSnapshots, *vs)
-			}
-		}
-
-		backup.CSISnapshots = volumeSnapshots
-
-		err = b.kbClient.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector})
-		if err != nil {
-			backupLog.Error(err)
-		}
-		if len(vscList.Items) >= 0 {
-			volumeSnapshotContents = vscList.Items
-		}
-
-		vsClassSet := sets.NewString()
-		for index := range volumeSnapshotContents {
-			// persist the volumesnapshotclasses referenced by vsc
-			if volumeSnapshotContents[index].Spec.VolumeSnapshotClassName != nil && !vsClassSet.Has(*volumeSnapshotContents[index].Spec.VolumeSnapshotClassName) {
-				vsClass := &snapshotv1api.VolumeSnapshotClass{}
-				if err := b.kbClient.Get(context.TODO(), kbclient.ObjectKey{Name: *volumeSnapshotContents[index].Spec.VolumeSnapshotClassName}, vsClass); err != nil {
-					backupLog.Error(err)
-				} else {
-					vsClassSet.Insert(*volumeSnapshotContents[index].Spec.VolumeSnapshotClassName)
-					volumeSnapshotClasses = append(volumeSnapshotClasses, *vsClass)
-				}
-			}
-		}
-	}
-
+	// native snapshots phase will either be failed or completed right away
+	// https://github.com/vmware-tanzu/velero/blob/de3ea52f0cc478e99efa7b9524c7f353514261a4/pkg/backup/item_backupper.go#L632-L639
 	backup.Status.VolumeSnapshotsAttempted = len(backup.VolumeSnapshots)
 	for _, snap := range backup.VolumeSnapshots {
 		if snap.Status.Phase == volume.SnapshotPhaseCompleted {
 			backup.Status.VolumeSnapshotsCompleted++
 		}
 	}
-
-	backup.Status.CSIVolumeSnapshotsAttempted = len(backup.CSISnapshots)
-	for _, vs := range backup.CSISnapshots {
-		if vs.Status != nil && boolptr.IsSetToTrue(vs.Status.ReadyToUse) {
-			backup.Status.CSIVolumeSnapshotsCompleted++
-		}
-	}
+	volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses := pkgbackup.UpdateBackupCSISnapshotsStatus(b.kbClient, b.volumeSnapshotLister, backup.Backup, backupLog)
 
 	// Iterate over backup item operations and update progress.
 	// Any errors on operations at this point should be added to backup errors.
@@ -848,6 +796,7 @@ func persistBackup(backup *pkgbackup.Request,
 ) []error {
 	persistErrs := []error{}
 	backupJSON := new(bytes.Buffer)
+	volumeInfos := make([]volume.VolumeInfo, 0)
 
 	if err := encode.To(backup.Backup, "json", backupJSON); err != nil {
 		persistErrs = append(persistErrs, errors.Wrap(err, "error encoding backup"))
@@ -894,6 +843,11 @@ func persistBackup(backup *pkgbackup.Request,
 		persistErrs = append(persistErrs, errs...)
 	}
 
+	volumeInfoJSON, errs := encode.ToJSONGzip(volumeInfos, "backup volumes information")
+	if errs != nil {
+		persistErrs = append(persistErrs, errs...)
+	}
+
 	if len(persistErrs) > 0 {
 		// Don't upload the JSON files or backup tarball if encoding to json fails.
 		backupJSON = nil
@@ -905,6 +859,7 @@ func persistBackup(backup *pkgbackup.Request,
 		csiSnapshotContentsJSON = nil
 		csiSnapshotClassesJSON = nil
 		backupResult = nil
+		volumeInfoJSON = nil
 	}
 
 	backupInfo := persistence.BackupInfo{
@@ -920,6 +875,7 @@ func persistBackup(backup *pkgbackup.Request,
 		CSIVolumeSnapshots:        csiSnapshotJSON,
 		CSIVolumeSnapshotContents: csiSnapshotContentsJSON,
 		CSIVolumeSnapshotClasses:  csiSnapshotClassesJSON,
+		BackupVolumeInfo:          volumeInfoJSON,
 	}
 	if err := backupStore.PutBackup(backupInfo); err != nil {
 		persistErrs = append(persistErrs, err)

pkg/controller/backup_controller_test.go

@@ -21,11 +21,13 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"reflect"
 	"sort"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	snapshotfake "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake"
 	snapshotinformers "github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions"
@@ -43,6 +45,10 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
+
+	fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/builder"
@@ -1665,3 +1671,63 @@ func Test_getLastSuccessBySchedule(t *testing.T) {
 		})
 	}
 }
+
+// Unit tests to make sure that the backup's status is updated correctly during reconcile.
+// To clear up confusion whether status can be updated with Patch alone without status writer and not kbClient.Status().Patch()
+func TestPatchResourceWorksWithStatus(t *testing.T) {
+	type args struct {
+		original *velerov1api.Backup
+		updated  *velerov1api.Backup
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "patch backup status",
+			args: args{
+				original: defaultBackup().SnapshotMoveData(false).Result(),
+				updated: defaultBackup().SnapshotMoveData(false).WithStatus(velerov1api.BackupStatus{
+					CSIVolumeSnapshotsCompleted: 1,
+				}).Result(),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scheme := runtime.NewScheme()
+			error := velerov1api.AddToScheme(scheme)
+			if error != nil {
+				t.Errorf("PatchResource() error = %v", error)
+			}
+			fakeClient := fakeClient.NewClientBuilder().WithScheme(scheme).WithObjects(tt.args.original).Build()
+			fromCluster := &velerov1api.Backup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      tt.args.original.Name,
+					Namespace: tt.args.original.Namespace,
+				},
+			}
+			// check original exists
+			if err := fakeClient.Get(context.Background(), kbclient.ObjectKeyFromObject(tt.args.updated), fromCluster); err != nil {
+				t.Errorf("PatchResource() error = %v", err)
+			}
+			// ignore resourceVersion
+			tt.args.updated.ResourceVersion = fromCluster.ResourceVersion
+			tt.args.original.ResourceVersion = fromCluster.ResourceVersion
+			if err := kubeutil.PatchResource(tt.args.original, tt.args.updated, fakeClient); (err != nil) != tt.wantErr {
+				t.Errorf("PatchResource() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			// check updated exists
+			if err := fakeClient.Get(context.Background(), kbclient.ObjectKeyFromObject(tt.args.updated), fromCluster); err != nil {
+				t.Errorf("PatchResource() error = %v", err)
+			}
+
+			// check fromCluster is equal to updated
+			if !reflect.DeepEqual(fromCluster, tt.args.updated) {
+				t.Error(cmp.Diff(fromCluster, tt.args.updated))
+			}
+		})
+
+	}
+}

pkg/controller/backup_finalizer_controller.go

@@ -29,6 +29,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
+
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/metrics"
@@ -40,19 +42,21 @@ import (
 
 // backupFinalizerReconciler reconciles a Backup object
 type backupFinalizerReconciler struct {
-	client            kbclient.Client
-	clock             clocks.WithTickerAndDelayedExecution
-	backupper         pkgbackup.Backupper
-	newPluginManager  func(logrus.FieldLogger) clientmgmt.Manager
-	backupTracker     BackupTracker
-	metrics           *metrics.ServerMetrics
-	backupStoreGetter persistence.ObjectBackupStoreGetter
-	log               logrus.FieldLogger
+	client               kbclient.Client
+	volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister
+	clock                clocks.WithTickerAndDelayedExecution
+	backupper            pkgbackup.Backupper
+	newPluginManager     func(logrus.FieldLogger) clientmgmt.Manager
+	backupTracker        BackupTracker
+	metrics              *metrics.ServerMetrics
+	backupStoreGetter    persistence.ObjectBackupStoreGetter
+	log                  logrus.FieldLogger
 }
 
 // NewBackupFinalizerReconciler initializes and returns backupFinalizerReconciler struct.
 func NewBackupFinalizerReconciler(
 	client kbclient.Client,
+	volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister,
 	clock clocks.WithTickerAndDelayedExecution,
 	backupper pkgbackup.Backupper,
 	newPluginManager func(logrus.FieldLogger) clientmgmt.Manager,
@@ -187,6 +191,7 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	backup.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()}
 	recordBackupMetrics(log, backup, outBackupFile, r.metrics, true)
 
+	pkgbackup.UpdateBackupCSISnapshotsStatus(r.client, r.volumeSnapshotLister, backup, log)
 	// update backup metadata in object store
 	backupJSON := new(bytes.Buffer)
 	if err := encode.To(backup, "json", backupJSON); err != nil {

pkg/controller/backup_finalizer_controller_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -43,12 +44,14 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
+	velerotestmocks "github.com/vmware-tanzu/velero/pkg/test/mocks"
 )
 
-func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) {
+func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeVolumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) {
 	backupper := new(fakeBackupper)
 	return NewBackupFinalizerReconciler(
 		fakeClient,
+		fakeVolumeSnapshotLister,
 		fakeClock,
 		backupper,
 		func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager },
@@ -160,7 +163,10 @@ func TestBackupFinalizerReconcile(t *testing.T) {
 			}
 
 			fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...)
-			reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeClock)
+
+			fakeVolumeSnapshotLister := velerotestmocks.NewVolumeSnapshotLister(t)
+
+			reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeVolumeSnapshotLister, fakeClock)
 			pluginManager.On("CleanupClients").Return(nil)
 			backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil)
 			backupStore.On("GetBackupContents", mock.Anything).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil)

pkg/controller/backup_operations_controller.go

@@ -19,8 +19,11 @@ package controller
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"time"
 
+	v2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2"
+
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -275,6 +278,8 @@ func (c *backupOperationsReconciler) updateBackupAndOperationsJSON(
 	return nil
 }
 
+// check progress of backupItemOperations
+// return: inProgressOperations, changes, completedCount, failedCount, errs
 func getBackupItemOperationProgress(
 	backup *velerov1api.Backup,
 	pluginManager clientmgmt.Manager,
@@ -291,7 +296,7 @@ func getBackupItemOperationProgress(
 			if err != nil {
 				operation.Status.Phase = itemoperation.OperationPhaseFailed
 				operation.Status.Error = err.Error()
-				errs = append(errs, err.Error())
+				errs = append(errs, wrapErrMsg(err.Error(), bia))
 				changes = true
 				failedCount++
 				continue
@@ -300,7 +305,7 @@ func getBackupItemOperationProgress(
 			if err != nil {
 				operation.Status.Phase = itemoperation.OperationPhaseFailed
 				operation.Status.Error = err.Error()
-				errs = append(errs, err.Error())
+				errs = append(errs, wrapErrMsg(err.Error(), bia))
 				changes = true
 				failedCount++
 				continue
@@ -338,7 +343,7 @@ func getBackupItemOperationProgress(
 				if operationProgress.Err != "" {
 					operation.Status.Phase = itemoperation.OperationPhaseFailed
 					operation.Status.Error = operationProgress.Err
-					errs = append(errs, operationProgress.Err)
+					errs = append(errs, wrapErrMsg(operationProgress.Err, bia))
 					changes = true
 					failedCount++
 					continue
@@ -353,7 +358,7 @@ func getBackupItemOperationProgress(
 				_ = bia.Cancel(operation.Spec.OperationID, backup)
 				operation.Status.Phase = itemoperation.OperationPhaseFailed
 				operation.Status.Error = "Asynchronous action timed out"
-				errs = append(errs, operation.Status.Error)
+				errs = append(errs, wrapErrMsg(operation.Status.Error, bia))
 				changes = true
 				failedCount++
 				continue
@@ -373,3 +378,15 @@ func getBackupItemOperationProgress(
 	}
 	return inProgressOperations, changes, completedCount, failedCount, errs
 }
+
+// wrap the error message to include the BIA name
+func wrapErrMsg(errMsg string, bia v2.BackupItemAction) string {
+	plugin := "unknown"
+	if bia != nil {
+		plugin = bia.Name()
+	}
+	if len(errMsg) > 0 {
+		errMsg += ", "
+	}
+	return fmt.Sprintf("%splugin: %s", errMsg, plugin)
+}

pkg/controller/backup_operations_controller_test.go

@@ -21,6 +21,8 @@ import (
 	"testing"
 	"time"
 
+	v2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2"
+
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -286,6 +288,7 @@ func TestBackupOperationsReconcile(t *testing.T) {
 			backupStore.On("PutBackupItemOperations", mock.Anything, mock.Anything).Return(nil)
 			backupStore.On("PutBackupMetadata", mock.Anything, mock.Anything).Return(nil)
 			for _, operation := range test.backupOperations {
+				bia.On("Name").Return("test")
 				bia.On("Progress", operation.Spec.OperationID, mock.Anything).
 					Return(velero.OperationProgress{
 						Completed: test.operationComplete,
@@ -308,3 +311,40 @@ func TestBackupOperationsReconcile(t *testing.T) {
 		})
 	}
 }
+
+func TestWrapErrMsg(t *testing.T) {
+	bia2 := &biav2mocks.BackupItemAction{}
+	bia2.On("Name").Return("test-bia")
+	cases := []struct {
+		name     string
+		inputErr string
+		plugin   v2.BackupItemAction
+		expect   string
+	}{
+		{
+			name:     "empty error message",
+			inputErr: "",
+			plugin:   bia2,
+			expect:   "plugin: test-bia",
+		},
+		{
+			name:     "nil bia",
+			inputErr: "some error happened",
+			plugin:   nil,
+			expect:   "some error happened, plugin: unknown",
+		},
+		{
+			name:     "regular error and bia",
+			inputErr: "some error happened",
+			plugin:   bia2,
+			expect:   "some error happened, plugin: test-bia",
+		},
+	}
+
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			got := wrapErrMsg(test.inputErr, test.plugin)
+			assert.Equal(t, test.expect, got)
+		})
+	}
+}

pkg/controller/backup_sync_controller.go

@@ -29,6 +29,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 
@@ -142,6 +143,16 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		log = log.WithField("backup", backupName)
 		log.Info("Attempting to sync backup into cluster")
 
+		exist, err := backupStore.BackupExists(location.Spec.ObjectStorage.Bucket, backupName)
+		if err != nil {
+			log.WithError(errors.WithStack(err)).Error("Error checking backup exist from backup store")
+			continue
+		}
+		if !exist {
+			log.Debugf("backup %s doesn't exist in backup store, skip", backupName)
+			continue
+		}
+
 		backup, err := backupStore.GetBackupMetadata(backupName)
 		if err != nil {
 			log.WithError(errors.WithStack(err)).Error("Error getting backup metadata from backup store")
@@ -171,6 +182,9 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		}
 		backup.Labels[velerov1api.StorageLocationLabel] = label.GetValidName(backup.Spec.StorageLocation)
 
+		//check for the ownership references. If they do not exist, remove them.
+		backup.ObjectMeta.OwnerReferences = b.filterBackupOwnerReferences(ctx, backup, log)
+
 		// attempt to create backup custom resource via API
 		err = b.client.Create(ctx, backup, &client.CreateOptions{})
 		switch {
@@ -286,6 +300,35 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	return ctrl.Result{}, nil
 }
 
+func (b *backupSyncReconciler) filterBackupOwnerReferences(ctx context.Context, backup *velerov1api.Backup, log logrus.FieldLogger) []metav1.OwnerReference {
+	listedReferences := backup.ObjectMeta.OwnerReferences
+	foundReferences := make([]metav1.OwnerReference, 0)
+	for _, v := range listedReferences {
+		switch v.Kind {
+		case "Schedule":
+			schedule := new(velerov1api.Schedule)
+			err := b.client.Get(ctx, types.NamespacedName{
+				Name:      v.Name,
+				Namespace: backup.Namespace,
+			}, schedule)
+			switch {
+			case err != nil && apierrors.IsNotFound(err):
+				log.Warnf("Removing missing schedule ownership reference %s/%s from backup", backup.Namespace, v.Name)
+				continue
+			case schedule.UID != v.UID:
+				log.Warnf("Removing schedule ownership reference with mismatched UIDs. Expected %s, got %s", v.UID, schedule.UID)
+				continue
+			case err != nil && !apierrors.IsNotFound(err):
+				log.WithError(errors.WithStack(err)).Error("Error finding schedule ownership reference, keeping schedule on backup")
+			}
+		default:
+			log.Warnf("Unable to check ownership reference for unknown kind, %s", v.Kind)
+		}
+		foundReferences = append(foundReferences, v)
+	}
+	return foundReferences
+}
+
 // SetupWithManager is used to setup controller and its watching sources.
 func (b *backupSyncReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	backupSyncSource := kube.NewPeriodicalEnqueueSource(

pkg/controller/backup_sync_controller_test.go

@@ -429,6 +429,7 @@ var _ = Describe("Backup Sync Reconciler", func() {
 					backupNames = append(backupNames, backup.backup.Name)
 					backupStore.On("GetBackupMetadata", backup.backup.Name).Return(backup.backup, nil)
 					backupStore.On("GetPodVolumeBackups", backup.backup.Name).Return(backup.podVolumeBackups, nil)
+					backupStore.On("BackupExists", "bucket-1", backup.backup.Name).Return(true, nil)
 				}
 				backupStore.On("ListBackups").Return(backupNames, nil)
 			}
@@ -724,4 +725,174 @@ var _ = Describe("Backup Sync Reconciler", func() {
 		Expect(testObjList).To(BeEquivalentTo(locationList))
 
 	})
+
+	When("testing validateOwnerReferences", func() {
+
+		testCases := []struct {
+			name               string
+			backup             *velerov1api.Backup
+			toCreate           []ctrlClient.Object
+			expectedReferences []metav1.OwnerReference
+		}{
+			{
+				name: "handles empty owner references",
+				backup: &velerov1api.Backup{
+					ObjectMeta: metav1.ObjectMeta{
+						OwnerReferences: []metav1.OwnerReference{},
+					},
+				},
+				expectedReferences: []metav1.OwnerReference{},
+			},
+			{
+				name: "handles missing schedule",
+				backup: &velerov1api.Backup{
+					ObjectMeta: metav1.ObjectMeta{
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								Kind: "Schedule",
+								Name: "some name",
+							},
+						},
+					},
+				},
+				expectedReferences: []metav1.OwnerReference{},
+			},
+			{
+				name: "handles existing reference",
+				backup: &velerov1api.Backup{
+					ObjectMeta: metav1.ObjectMeta{
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								Kind: "Schedule",
+								Name: "existing-schedule",
+							},
+						},
+						Namespace: "test-namespace",
+					},
+				},
+				toCreate: []ctrlClient.Object{
+					&velerov1api.Schedule{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "existing-schedule",
+							Namespace: "test-namespace",
+						},
+					},
+				},
+				expectedReferences: []metav1.OwnerReference{
+					{
+						Kind: "Schedule",
+						Name: "existing-schedule",
+					},
+				},
+			},
+			{
+				name: "handles existing mismatched UID",
+				backup: &velerov1api.Backup{
+					ObjectMeta: metav1.ObjectMeta{
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								Kind: "Schedule",
+								Name: "existing-schedule",
+								UID:  "backup-UID",
+							},
+						},
+						Namespace: "test-namespace",
+					},
+				},
+				toCreate: []ctrlClient.Object{
+					&velerov1api.Schedule{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "existing-schedule",
+							Namespace: "test-namespace",
+							UID:       "schedule-UID",
+						},
+					},
+				},
+				expectedReferences: []metav1.OwnerReference{},
+			},
+			{
+				name: "handles multiple references",
+				backup: &velerov1api.Backup{
+					ObjectMeta: metav1.ObjectMeta{
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								Kind: "Schedule",
+								Name: "existing-schedule",
+								UID:  "1",
+							},
+							{
+								Kind: "Schedule",
+								Name: "missing-schedule",
+								UID:  "2",
+							},
+							{
+								Kind: "Schedule",
+								Name: "mismatched-uid-schedule",
+								UID:  "3",
+							},
+							{
+								Kind: "Schedule",
+								Name: "another-existing-schedule",
+								UID:  "4",
+							},
+						},
+						Namespace: "test-namespace",
+					},
+				},
+				toCreate: []ctrlClient.Object{
+					&velerov1api.Schedule{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "existing-schedule",
+							Namespace: "test-namespace",
+							UID:       "1",
+						},
+					},
+					&velerov1api.Schedule{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "mismatched-uid-schedule",
+							Namespace: "test-namespace",
+							UID:       "not-3",
+						},
+					},
+					&velerov1api.Schedule{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "another-existing-schedule",
+							Namespace: "test-namespace",
+							UID:       "4",
+						},
+					},
+				},
+				expectedReferences: []metav1.OwnerReference{
+					{
+						Kind: "Schedule",
+						Name: "existing-schedule",
+						UID:  "1",
+					},
+					{
+						Kind: "Schedule",
+						Name: "another-existing-schedule",
+						UID:  "4",
+					},
+				},
+			},
+		}
+		for _, test := range testCases {
+			test := test
+			It(test.name, func() {
+				logger := velerotest.NewLogger()
+				b := backupSyncReconciler{
+					client: ctrlfake.NewClientBuilder().Build(),
+				}
+
+				//create all required schedules as needed.
+				for _, creatable := range test.toCreate {
+					err := b.client.Create(context.Background(), creatable)
+					Expect(err).ShouldNot(HaveOccurred())
+				}
+
+				references := b.filterBackupOwnerReferences(context.Background(), test.backup, logger)
+				Expect(references).To(BeEquivalentTo(test.expectedReferences))
+			})
+		}
+	})
 })

pkg/controller/data_download_controller.go

@@ -69,7 +69,7 @@ type DataDownloadReconciler struct {
 	metrics           *metrics.ServerMetrics
 }
 
-func NewDataDownloadReconciler(client client.Client, kubeClient kubernetes.Interface,
+func NewDataDownloadReconciler(client client.Client, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager,
 	repoEnsurer *repository.Ensurer, credentialGetter *credentials.CredentialGetter, nodeName string, preparingTimeout time.Duration, logger logrus.FieldLogger, metrics *metrics.ServerMetrics) *DataDownloadReconciler {
 	return &DataDownloadReconciler{
 		client:            client,
@@ -81,7 +81,7 @@ func NewDataDownloadReconciler(client client.Client, kubeClient kubernetes.Inter
 		nodeName:          nodeName,
 		repositoryEnsurer: repoEnsurer,
 		restoreExposer:    exposer.NewGenericRestoreExposer(kubeClient, logger),
-		dataPathMgr:       datapath.NewManager(1),
+		dataPathMgr:       dataPathMgr,
 		preparingTimeout:  preparingTimeout,
 		metrics:           metrics,
 	}
@@ -287,12 +287,13 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	} else if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress {
 		log.Info("Data download is in progress")
 		if dd.Spec.Cancel {
+			log.Info("Data download is being canceled")
 			fsRestore := r.dataPathMgr.GetAsyncBR(dd.Name)
 			if fsRestore == nil {
+				r.OnDataDownloadCancelled(ctx, dd.GetNamespace(), dd.GetName())
 				return ctrl.Result{}, nil
 			}
 
-			log.Info("Data download is being canceled")
 			// Update status to Canceling.
 			original := dd.DeepCopy()
 			dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseCanceling
@@ -300,7 +301,6 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
 				log.WithError(err).Error("error updating data download status")
 				return ctrl.Result{}, err
 			}
-
 			fsRestore.Cancel()
 			return ctrl.Result{}, nil
 		}

pkg/controller/data_download_controller_test.go

@@ -142,7 +142,10 @@ func initDataDownloadReconcilerWithError(objects []runtime.Object, needError ...
 	if err != nil {
 		return nil, err
 	}
-	return NewDataDownloadReconciler(fakeClient, fakeKubeClient, nil, &credentials.CredentialGetter{FromFile: credentialFileStore}, "test_node", time.Minute*5, velerotest.NewLogger(), metrics.NewServerMetrics()), nil
+
+	dataPathMgr := datapath.NewManager(1)
+
+	return NewDataDownloadReconciler(fakeClient, fakeKubeClient, dataPathMgr, nil, &credentials.CredentialGetter{FromFile: credentialFileStore}, "test_node", time.Minute*5, velerotest.NewLogger(), metrics.NewServerMetrics()), nil
 }
 
 func TestDataDownloadReconcile(t *testing.T) {

pkg/controller/data_upload_controller.go

@@ -79,8 +79,8 @@ type DataUploadReconciler struct {
 	metrics             *metrics.ServerMetrics
 }
 
-func NewDataUploadReconciler(client client.Client, kubeClient kubernetes.Interface,
-	csiSnapshotClient snapshotter.SnapshotV1Interface, repoEnsurer *repository.Ensurer, clock clocks.WithTickerAndDelayedExecution,
+func NewDataUploadReconciler(client client.Client, kubeClient kubernetes.Interface, csiSnapshotClient snapshotter.SnapshotV1Interface,
+	dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, clock clocks.WithTickerAndDelayedExecution,
 	cred *credentials.CredentialGetter, nodeName string, fs filesystem.Interface, preparingTimeout time.Duration, log logrus.FieldLogger, metrics *metrics.ServerMetrics) *DataUploadReconciler {
 	return &DataUploadReconciler{
 		client:              client,
@@ -93,7 +93,7 @@ func NewDataUploadReconciler(client client.Client, kubeClient kubernetes.Interfa
 		logger:              log,
 		repoEnsurer:         repoEnsurer,
 		snapshotExposerList: map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer(kubeClient, csiSnapshotClient, log)},
-		dataPathMgr:         datapath.NewManager(1),
+		dataPathMgr:         dataPathMgr,
 		preparingTimeout:    preparingTimeout,
 		metrics:             metrics,
 	}
@@ -292,13 +292,15 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	} else if du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress {
 		log.Info("Data upload is in progress")
 		if du.Spec.Cancel {
-			fsBackup := r.dataPathMgr.GetAsyncBR(du.Name)
-			if fsBackup == nil {
-				return ctrl.Result{}, nil
-			}
 			log.Info("Data upload is being canceled")
 
-			// Update status to Canceling.
+			fsBackup := r.dataPathMgr.GetAsyncBR(du.Name)
+			if fsBackup == nil {
+				r.OnDataUploadCancelled(ctx, du.GetNamespace(), du.GetName())
+				return ctrl.Result{}, nil
+			}
+
+			// Update status to Canceling
 			original := du.DeepCopy()
 			du.Status.Phase = velerov2alpha1api.DataUploadPhaseCanceling
 			if err := r.client.Patch(ctx, du, client.MergeFrom(original)); err != nil {
@@ -761,7 +763,9 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload
 			StorageClass:     du.Spec.CSISnapshot.StorageClass,
 			HostingPodLabels: map[string]string{velerov1api.DataUploadLabel: du.Name},
 			AccessMode:       accessMode,
-			Timeout:          du.Spec.OperationTimeout.Duration,
+			OperationTimeout: du.Spec.OperationTimeout.Duration,
+			ExposeTimeout:    r.preparingTimeout,
+			VolumeSize:       pvc.Spec.Resources.Requests[corev1.ResourceStorage],
 		}, nil
 	}
 	return nil, nil

pkg/controller/data_upload_controller_test.go

@@ -163,6 +163,8 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci
 		Spec: appsv1.DaemonSetSpec{},
 	}
 
+	dataPathMgr := datapath.NewManager(1)
+
 	now, err := time.Parse(time.RFC1123, time.RFC1123)
 	if err != nil {
 		return nil, err
@@ -218,7 +220,7 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci
 	if err != nil {
 		return nil, err
 	}
-	return NewDataUploadReconciler(fakeClient, fakeKubeClient, fakeSnapshotClient.SnapshotV1(), nil,
+	return NewDataUploadReconciler(fakeClient, fakeKubeClient, fakeSnapshotClient.SnapshotV1(), dataPathMgr, nil,
 		testclocks.NewFakeClock(now), &credentials.CredentialGetter{FromFile: credentialFileStore}, "test_node", fakeFS, time.Minute*5, velerotest.NewLogger(), metrics.NewServerMetrics()), nil
 }
 

pkg/controller/pod_volume_backup_controller.go

@@ -47,7 +47,7 @@ import (
 const pVBRRequestor string = "pod-volume-backup-restore"
 
 // NewPodVolumeBackupReconciler creates the PodVolumeBackupReconciler instance
-func NewPodVolumeBackupReconciler(client client.Client, ensurer *repository.Ensurer, credentialGetter *credentials.CredentialGetter,
+func NewPodVolumeBackupReconciler(client client.Client, dataPathMgr *datapath.Manager, ensurer *repository.Ensurer, credentialGetter *credentials.CredentialGetter,
 	nodeName string, scheme *runtime.Scheme, metrics *metrics.ServerMetrics, logger logrus.FieldLogger) *PodVolumeBackupReconciler {
 	return &PodVolumeBackupReconciler{
 		Client:            client,
@@ -59,7 +59,7 @@ func NewPodVolumeBackupReconciler(client client.Client, ensurer *repository.Ensu
 		clock:             &clocks.RealClock{},
 		scheme:            scheme,
 		metrics:           metrics,
-		dataPathMgr:       datapath.NewManager(1),
+		dataPathMgr:       dataPathMgr,
 	}
 }
 

pkg/controller/pod_volume_restore_controller.go

@@ -50,7 +50,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
 )
 
-func NewPodVolumeRestoreReconciler(client client.Client, ensurer *repository.Ensurer,
+func NewPodVolumeRestoreReconciler(client client.Client, dataPathMgr *datapath.Manager, ensurer *repository.Ensurer,
 	credentialGetter *credentials.CredentialGetter, logger logrus.FieldLogger) *PodVolumeRestoreReconciler {
 	return &PodVolumeRestoreReconciler{
 		Client:            client,
@@ -59,7 +59,7 @@ func NewPodVolumeRestoreReconciler(client client.Client, ensurer *repository.Ens
 		credentialGetter:  credentialGetter,
 		fileSystem:        filesystem.NewFileSystem(),
 		clock:             &clocks.RealClock{},
-		dataPathMgr:       datapath.NewManager(1),
+		dataPathMgr:       dataPathMgr,
 	}
 }
 

pkg/controller/restore_controller.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"os"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -376,7 +377,7 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf
 	}
 
 	var resourceModifiers *resourcemodifiers.ResourceModifiers = nil
-	if restore.Spec.ResourceModifier != nil && restore.Spec.ResourceModifier.Kind == resourcemodifiers.ConfigmapRefType {
+	if restore.Spec.ResourceModifier != nil && strings.EqualFold(restore.Spec.ResourceModifier.Kind, resourcemodifiers.ConfigmapRefType) {
 		ResourceModifierConfigMap := &corev1api.ConfigMap{}
 		err := r.kbClient.Get(context.Background(), client.ObjectKey{Namespace: restore.Namespace, Name: restore.Spec.ResourceModifier.Name}, ResourceModifierConfigMap)
 		if err != nil {
@@ -514,6 +515,11 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu
 		return errors.Wrap(err, "error fetching volume snapshots metadata")
 	}
 
+	csiVolumeSnapshots, err := backupStore.GetCSIVolumeSnapshots(restore.Spec.BackupName)
+	if err != nil {
+		return errors.Wrap(err, "fail to fetch CSI VolumeSnapshots metadata")
+	}
+
 	restoreLog.Info("starting restore")
 
 	var podVolumeBackups []*api.PodVolumeBackup
@@ -530,6 +536,7 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu
 		BackupReader:         backupFile,
 		ResourceModifiers:    resourceModifiers,
 		DisableInformerCache: r.disableInformerCache,
+		CSIVolumeSnapshots:   csiVolumeSnapshots,
 	}
 	restoreWarnings, restoreErrors := r.restorer.RestoreWithResolvers(restoreReq, actionsResolver, pluginManager)
 

pkg/controller/restore_controller_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -471,6 +472,7 @@ func TestRestoreReconcile(t *testing.T) {
 			}
 			if test.expectedRestorerCall != nil {
 				backupStore.On("GetBackupContents", test.backup.Name).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil)
+				backupStore.On("GetCSIVolumeSnapshots", test.backup.Name).Return([]*snapshotv1api.VolumeSnapshot{}, nil)
 
 				restorer.On("RestoreWithResolvers", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
 					mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors)
@@ -781,7 +783,8 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) {
 		Spec: velerov1api.RestoreSpec{
 			BackupName: "backup-1",
 			ResourceModifier: &corev1.TypedLocalObjectReference{
-				Kind: resourcemodifiers.ConfigmapRefType,
+				// intentional to ensure case insensitivity works as expected
+				Kind: "confIGMaP",
 				Name: "test-configmap-invalid",
 			},
 		},

pkg/discovery/helper_test.go

@@ -30,7 +30,6 @@ import (
 	clientgotesting "k8s.io/client-go/testing"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
-	discoverymocks "github.com/vmware-tanzu/velero/pkg/discovery/mocks"
 	"github.com/vmware-tanzu/velero/pkg/features"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
 	"github.com/vmware-tanzu/velero/pkg/util/logging"
@@ -548,39 +547,31 @@ func TestHelper_refreshServerPreferredResources(t *testing.T) {
 	}
 
 	tests := []struct {
-		name          string
-		isGetResError bool
+		name        string
+		expectedErr error
 	}{
 		{
-			name: "success get preferred resources",
+			name:        "success get preferred resources",
+			expectedErr: nil,
 		},
 		{
-			name:          "failed to get preferred resources",
-			isGetResError: true,
+			name:        "failed to get preferred resources",
+			expectedErr: errors.New("Failed to discover preferred resources"),
 		},
 	}
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			fakeClient := discoverymocks.NewServerResourcesInterface(t)
-
-			if tc.isGetResError {
-				fakeClient.On("ServerPreferredResources").Return(nil, errors.New("Failed to discover preferred resources"))
-			} else {
-				fakeClient.On("ServerPreferredResources").Return(apiList, nil)
-			}
+			fakeClient := velerotest.NewFakeServerResourcesInterface(apiList, []*metav1.APIGroup{}, map[schema.GroupVersion]error{}, tc.expectedErr)
 
 			resources, err := refreshServerPreferredResources(fakeClient, logrus.New())
 
-			if tc.isGetResError {
+			if tc.expectedErr != nil {
 				assert.NotNil(t, err)
-				assert.Nil(t, resources)
 			} else {
 				assert.Nil(t, err)
 				assert.NotNil(t, resources)
 			}
-
-			fakeClient.AssertExpectations(t)
 		})
 	}
 }
@@ -612,41 +603,31 @@ func TestHelper_refreshServerGroupsAndResources(t *testing.T) {
 		},
 	}
 	tests := []struct {
-		name          string
-		isGetResError bool
+		name        string
+		expectedErr error
 	}{
 		{
 			name: "success get service groups and resouorces",
 		},
 		{
-			name:          "failed to service groups and resouorces",
-			isGetResError: true,
+			name:        "failed to service groups and resouorces",
+			expectedErr: errors.New("Failed to discover service groups and resouorces"),
 		},
 	}
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			fakeClient := discoverymocks.NewServerResourcesInterface(t)
-
-			if tc.isGetResError {
-				fakeClient.On("ServerGroupsAndResources").Return(nil, nil, errors.New("Failed to discover service groups and resouorces"))
-			} else {
-				fakeClient.On("ServerGroupsAndResources").Return(apiGroup, apiList, nil)
-			}
+			fakeClient := velerotest.NewFakeServerResourcesInterface(apiList, apiGroup, map[schema.GroupVersion]error{}, tc.expectedErr)
 
 			serverGroups, serverResources, err := refreshServerGroupsAndResources(fakeClient, logrus.New())
 
-			if tc.isGetResError {
+			if tc.expectedErr != nil {
 				assert.NotNil(t, err)
-				assert.Nil(t, serverGroups)
-				assert.Nil(t, serverResources)
 			} else {
 				assert.Nil(t, err)
 				assert.NotNil(t, serverGroups)
 				assert.NotNil(t, serverResources)
 			}
-
-			fakeClient.AssertExpectations(t)
 		})
 	}
 }

pkg/discovery/mocks/Helper.go

@@ -1,4 +1,4 @@
-// Code generated by mockery v2.30.1. DO NOT EDIT.
+// Code generated by mockery v2.20.0. DO NOT EDIT.
 
 package mocks
 
@@ -140,12 +140,13 @@ func (_m *Helper) ServerVersion() *version.Info {
 	return r0
 }
 
-// NewHelper creates a new instance of Helper. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
-// The first argument is typically a *testing.T value.
-func NewHelper(t interface {
+type mockConstructorTestingTNewHelper interface {
 	mock.TestingT
 	Cleanup(func())
-}) *Helper {
+}
+
+// NewHelper creates a new instance of Helper. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewHelper(t mockConstructorTestingTNewHelper) *Helper {
 	mock := &Helper{}
 	mock.Mock.Test(t)
 
@@ -153,84 +154,3 @@ func NewHelper(t interface {
 
 	return mock
 }
-
-// serverResourcesInterface is an autogenerated mock type for the serverResourcesInterface type
-type serverResourcesInterface struct {
-	mock.Mock
-}
-
-// ServerGroupsAndResources provides a mock function with given fields:
-func (_m *serverResourcesInterface) ServerGroupsAndResources() ([]*v1.APIGroup, []*v1.APIResourceList, error) {
-	ret := _m.Called()
-
-	var r0 []*v1.APIGroup
-	var r1 []*v1.APIResourceList
-	var r2 error
-	if rf, ok := ret.Get(0).(func() ([]*v1.APIGroup, []*v1.APIResourceList, error)); ok {
-		return rf()
-	}
-	if rf, ok := ret.Get(0).(func() []*v1.APIGroup); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]*v1.APIGroup)
-		}
-	}
-
-	if rf, ok := ret.Get(1).(func() []*v1.APIResourceList); ok {
-		r1 = rf()
-	} else {
-		if ret.Get(1) != nil {
-			r1 = ret.Get(1).([]*v1.APIResourceList)
-		}
-	}
-
-	if rf, ok := ret.Get(2).(func() error); ok {
-		r2 = rf()
-	} else {
-		r2 = ret.Error(2)
-	}
-
-	return r0, r1, r2
-}
-
-// ServerPreferredResources provides a mock function with given fields:
-func (_m *serverResourcesInterface) ServerPreferredResources() ([]*v1.APIResourceList, error) {
-	ret := _m.Called()
-
-	var r0 []*v1.APIResourceList
-	var r1 error
-	if rf, ok := ret.Get(0).(func() ([]*v1.APIResourceList, error)); ok {
-		return rf()
-	}
-	if rf, ok := ret.Get(0).(func() []*v1.APIResourceList); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]*v1.APIResourceList)
-		}
-	}
-
-	if rf, ok := ret.Get(1).(func() error); ok {
-		r1 = rf()
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}
-
-type mockConstructorTestingTnewServerResourcesInterface interface {
-	mock.TestingT
-	Cleanup(func())
-}
-
-// NewServerResourcesInterface creates a new instance of serverResourcesInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
-func NewServerResourcesInterface(t mockConstructorTestingTnewServerResourcesInterface) *serverResourcesInterface {
-	mock := &serverResourcesInterface{}
-	mock.Mock.Test(t)
-
-	t.Cleanup(func() { mock.AssertExpectations(t) })
-
-	return mock
-}

pkg/exposer/csi_snapshot.go

@@ -59,8 +59,14 @@ type CSISnapshotExposeParam struct {
 	// HostingPodLabels is the labels that are going to apply to the hosting pod
 	HostingPodLabels map[string]string
 
-	// Timeout specifies the time wait for resources operations in Expose
-	Timeout time.Duration
+	// OperationTimeout specifies the time wait for resources operations in Expose
+	OperationTimeout time.Duration
+
+	// ExposeTimeout specifies the timeout for the entire expose process
+	ExposeTimeout time.Duration
+
+	// VolumeSize specifies the size of the source volume
+	VolumeSize resource.Quantity
 }
 
 // CSISnapshotExposeWaitParam define the input param for WaitExposed of CSI snapshots
@@ -94,7 +100,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1.Obje
 
 	curLog.Info("Exposing CSI snapshot")
 
-	volumeSnapshot, err := csi.WaitVolumeSnapshotReady(ctx, e.csiSnapshotClient, csiExposeParam.SnapshotName, csiExposeParam.SourceNamespace, csiExposeParam.Timeout, curLog)
+	volumeSnapshot, err := csi.WaitVolumeSnapshotReady(ctx, e.csiSnapshotClient, csiExposeParam.SnapshotName, csiExposeParam.SourceNamespace, csiExposeParam.ExposeTimeout, curLog)
 	if err != nil {
 		return errors.Wrapf(err, "error wait volume snapshot ready")
 	}
@@ -121,14 +127,21 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1.Obje
 		}
 	}()
 
-	err = csi.EnsureDeleteVS(ctx, e.csiSnapshotClient, volumeSnapshot.Name, volumeSnapshot.Namespace, csiExposeParam.Timeout)
+	err = csi.EnsureDeleteVS(ctx, e.csiSnapshotClient, volumeSnapshot.Name, volumeSnapshot.Namespace, csiExposeParam.OperationTimeout)
 	if err != nil {
 		return errors.Wrap(err, "error to delete volume snapshot")
 	}
 
 	curLog.WithField("vs name", volumeSnapshot.Name).Infof("VS is deleted in namespace %s", volumeSnapshot.Namespace)
 
-	err = csi.EnsureDeleteVSC(ctx, e.csiSnapshotClient, vsc.Name, csiExposeParam.Timeout)
+	err = csi.RemoveVSCProtect(ctx, e.csiSnapshotClient, vsc.Name, csiExposeParam.ExposeTimeout)
+	if err != nil {
+		return errors.Wrap(err, "error to remove protect from volume snapshot content")
+	}
+
+	curLog.WithField("vsc name", vsc.Name).Infof("Removed protect from VSC")
+
+	err = csi.EnsureDeleteVSC(ctx, e.csiSnapshotClient, vsc.Name, csiExposeParam.OperationTimeout)
 	if err != nil {
 		return errors.Wrap(err, "error to delete volume snapshot content")
 	}
@@ -156,7 +169,15 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1.Obje
 
 	curLog.WithField("vsc name", backupVSC.Name).Infof("Backup VSC is created from %s", vsc.Name)
 
-	backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, csiExposeParam.StorageClass, csiExposeParam.AccessMode, *volumeSnapshot.Status.RestoreSize)
+	var volumeSize resource.Quantity
+	if volumeSnapshot.Status.RestoreSize != nil && !volumeSnapshot.Status.RestoreSize.IsZero() {
+		volumeSize = *volumeSnapshot.Status.RestoreSize
+	} else {
+		volumeSize = csiExposeParam.VolumeSize
+		curLog.WithField("vs name", volumeSnapshot.Name).Warnf("The snapshot doesn't contain a valid restore size, use source volume's size %v", volumeSize)
+	}
+
+	backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, csiExposeParam.StorageClass, csiExposeParam.AccessMode, volumeSize)
 	if err != nil {
 		return errors.Wrap(err, "error to create backup pvc")
 	}
@@ -190,6 +211,7 @@ func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1.
 
 	backupPodName := ownerObject.Name
 	backupPVCName := ownerObject.Name
+	volumeName := string(ownerObject.UID)
 
 	curLog := e.log.WithFields(logrus.Fields{
 		"owner": ownerObject.Name,
@@ -218,7 +240,20 @@ func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1.
 
 	curLog.WithField("backup pvc", backupPVCName).Info("Backup PVC is bound")
 
-	return &ExposeResult{ByPod: ExposeByPod{HostingPod: pod, VolumeName: pod.Spec.Volumes[0].Name}}, nil
+	i := 0
+	for i = 0; i < len(pod.Spec.Volumes); i++ {
+		if pod.Spec.Volumes[i].Name == volumeName {
+			break
+		}
+	}
+
+	if i == len(pod.Spec.Volumes) {
+		return nil, errors.Errorf("backup pod %s doesn't have the expected backup volume", pod.Name)
+	}
+
+	curLog.WithField("pod", pod.Name).Infof("Backup volume is found in pod at index %v", i)
+
+	return &ExposeResult{ByPod: ExposeByPod{HostingPod: pod, VolumeName: volumeName}}, nil
 }
 
 func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1.ObjectReference, vsName string, sourceNamespace string) {

pkg/exposer/csi_snapshot_test.go

@@ -37,6 +37,8 @@ import (
 	velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
+
+	clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
 type reactor struct {
@@ -59,6 +61,8 @@ func TestExpose(t *testing.T) {
 		},
 	}
 
+	var restoreSize int64 = 123456
+
 	snapshotClass := "fake-snapshot-class"
 	vsObject := &snapshotv1api.VolumeSnapshot{
 		ObjectMeta: metav1.ObjectMeta{
@@ -78,11 +82,31 @@ func TestExpose(t *testing.T) {
 		Status: &snapshotv1api.VolumeSnapshotStatus{
 			BoundVolumeSnapshotContentName: &vscName,
 			ReadyToUse:                     boolptr.True(),
-			RestoreSize:                    &resource.Quantity{},
+			RestoreSize:                    resource.NewQuantity(restoreSize, ""),
+		},
+	}
+
+	vsObjectWithoutRestoreSize := &snapshotv1api.VolumeSnapshot{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "fake-vs",
+			Namespace: "fake-ns",
+			Annotations: map[string]string{
+				"fake-key-1": "fake-value-1",
+				"fake-key-2": "fake-value-2",
+			},
+		},
+		Spec: snapshotv1api.VolumeSnapshotSpec{
+			Source: snapshotv1api.VolumeSnapshotSource{
+				VolumeSnapshotContentName: &vscName,
+			},
+			VolumeSnapshotClassName: &snapshotClass,
+		},
+		Status: &snapshotv1api.VolumeSnapshotStatus{
+			BoundVolumeSnapshotContentName: &vscName,
+			ReadyToUse:                     boolptr.True(),
 		},
 	}
 
-	var restoreSize int64
 	snapshotHandle := "fake-handle"
 	vscObj := &snapshotv1api.VolumeSnapshotContent{
 		ObjectMeta: metav1.ObjectMeta{
@@ -116,21 +140,23 @@ func TestExpose(t *testing.T) {
 	}
 
 	tests := []struct {
-		name              string
-		snapshotClientObj []runtime.Object
-		kubeClientObj     []runtime.Object
-		ownerBackup       *velerov1.Backup
-		exposeParam       CSISnapshotExposeParam
-		snapReactors      []reactor
-		kubeReactors      []reactor
-		err               string
+		name               string
+		snapshotClientObj  []runtime.Object
+		kubeClientObj      []runtime.Object
+		ownerBackup        *velerov1.Backup
+		exposeParam        CSISnapshotExposeParam
+		snapReactors       []reactor
+		kubeReactors       []reactor
+		err                string
+		expectedVolumeSize *resource.Quantity
 	}{
 		{
 			name:        "wait vs ready fail",
 			ownerBackup: backup,
 			exposeParam: CSISnapshotExposeParam{
-				SnapshotName: "fake-vs",
-				Timeout:      time.Millisecond,
+				SnapshotName:     "fake-vs",
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
 			},
 			err: "error wait volume snapshot ready: error to get volumesnapshot /fake-vs: volumesnapshots.snapshot.storage.k8s.io \"fake-vs\" not found",
 		},
@@ -138,9 +164,10 @@ func TestExpose(t *testing.T) {
 			name:        "get vsc fail",
 			ownerBackup: backup,
 			exposeParam: CSISnapshotExposeParam{
-				SnapshotName:    "fake-vs",
-				SourceNamespace: "fake-ns",
-				Timeout:         time.Millisecond,
+				SnapshotName:     "fake-vs",
+				SourceNamespace:  "fake-ns",
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
 			},
 			snapshotClientObj: []runtime.Object{
 				vsObject,
@@ -151,9 +178,10 @@ func TestExpose(t *testing.T) {
 			name:        "delete vs fail",
 			ownerBackup: backup,
 			exposeParam: CSISnapshotExposeParam{
-				SnapshotName:    "fake-vs",
-				SourceNamespace: "fake-ns",
-				Timeout:         time.Millisecond,
+				SnapshotName:     "fake-vs",
+				SourceNamespace:  "fake-ns",
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
 			},
 			snapshotClientObj: []runtime.Object{
 				vsObject,
@@ -174,9 +202,10 @@ func TestExpose(t *testing.T) {
 			name:        "delete vsc fail",
 			ownerBackup: backup,
 			exposeParam: CSISnapshotExposeParam{
-				SnapshotName:    "fake-vs",
-				SourceNamespace: "fake-ns",
-				Timeout:         time.Millisecond,
+				SnapshotName:     "fake-vs",
+				SourceNamespace:  "fake-ns",
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
 			},
 			snapshotClientObj: []runtime.Object{
 				vsObject,
@@ -197,9 +226,10 @@ func TestExpose(t *testing.T) {
 			name:        "create backup vs fail",
 			ownerBackup: backup,
 			exposeParam: CSISnapshotExposeParam{
-				SnapshotName:    "fake-vs",
-				SourceNamespace: "fake-ns",
-				Timeout:         time.Millisecond,
+				SnapshotName:     "fake-vs",
+				SourceNamespace:  "fake-ns",
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
 			},
 			snapshotClientObj: []runtime.Object{
 				vsObject,
@@ -220,9 +250,10 @@ func TestExpose(t *testing.T) {
 			name:        "create backup vsc fail",
 			ownerBackup: backup,
 			exposeParam: CSISnapshotExposeParam{
-				SnapshotName:    "fake-vs",
-				SourceNamespace: "fake-ns",
-				Timeout:         time.Millisecond,
+				SnapshotName:     "fake-vs",
+				SourceNamespace:  "fake-ns",
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
 			},
 			snapshotClientObj: []runtime.Object{
 				vsObject,
@@ -257,10 +288,11 @@ func TestExpose(t *testing.T) {
 			name:        "create backup pvc fail",
 			ownerBackup: backup,
 			exposeParam: CSISnapshotExposeParam{
-				SnapshotName:    "fake-vs",
-				SourceNamespace: "fake-ns",
-				Timeout:         time.Millisecond,
-				AccessMode:      AccessModeFileSystem,
+				SnapshotName:     "fake-vs",
+				SourceNamespace:  "fake-ns",
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
+				AccessMode:       AccessModeFileSystem,
 			},
 			snapshotClientObj: []runtime.Object{
 				vsObject,
@@ -281,10 +313,11 @@ func TestExpose(t *testing.T) {
 			name:        "create backup pod fail",
 			ownerBackup: backup,
 			exposeParam: CSISnapshotExposeParam{
-				SnapshotName:    "fake-vs",
-				SourceNamespace: "fake-ns",
-				AccessMode:      AccessModeFileSystem,
-				Timeout:         time.Millisecond,
+				SnapshotName:     "fake-vs",
+				SourceNamespace:  "fake-ns",
+				AccessMode:       AccessModeFileSystem,
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
 			},
 			snapshotClientObj: []runtime.Object{
 				vsObject,
@@ -308,10 +341,11 @@ func TestExpose(t *testing.T) {
 			name:        "success",
 			ownerBackup: backup,
 			exposeParam: CSISnapshotExposeParam{
-				SnapshotName:    "fake-vs",
-				SourceNamespace: "fake-ns",
-				AccessMode:      AccessModeFileSystem,
-				Timeout:         time.Millisecond,
+				SnapshotName:     "fake-vs",
+				SourceNamespace:  "fake-ns",
+				AccessMode:       AccessModeFileSystem,
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
 			},
 			snapshotClientObj: []runtime.Object{
 				vsObject,
@@ -321,6 +355,26 @@ func TestExpose(t *testing.T) {
 				daemonSet,
 			},
 		},
+		{
+			name:        "restore size from exposeParam",
+			ownerBackup: backup,
+			exposeParam: CSISnapshotExposeParam{
+				SnapshotName:     "fake-vs",
+				SourceNamespace:  "fake-ns",
+				AccessMode:       AccessModeFileSystem,
+				OperationTimeout: time.Millisecond,
+				ExposeTimeout:    time.Millisecond,
+				VolumeSize:       *resource.NewQuantity(567890, ""),
+			},
+			snapshotClientObj: []runtime.Object{
+				vsObjectWithoutRestoreSize,
+				vscObj,
+			},
+			kubeClientObj: []runtime.Object{
+				daemonSet,
+			},
+			expectedVolumeSize: resource.NewQuantity(567890, ""),
+		},
 	}
 
 	for _, test := range tests {
@@ -360,7 +414,7 @@ func TestExpose(t *testing.T) {
 				_, err = exposer.kubeClient.CoreV1().Pods(ownerObject.Namespace).Get(context.Background(), ownerObject.Name, metav1.GetOptions{})
 				assert.NoError(t, err)
 
-				_, err = exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(context.Background(), ownerObject.Name, metav1.GetOptions{})
+				backupPVC, err := exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(context.Background(), ownerObject.Name, metav1.GetOptions{})
 				assert.NoError(t, err)
 
 				expectedVS, err := exposer.csiSnapshotClient.VolumeSnapshots(ownerObject.Namespace).Get(context.Background(), ownerObject.Name, metav1.GetOptions{})
@@ -377,6 +431,12 @@ func TestExpose(t *testing.T) {
 				assert.Equal(t, expectedVSC.Spec.DeletionPolicy, vscObj.Spec.DeletionPolicy)
 				assert.Equal(t, expectedVSC.Spec.Driver, vscObj.Spec.Driver)
 				assert.Equal(t, *expectedVSC.Spec.VolumeSnapshotClassName, *vscObj.Spec.VolumeSnapshotClassName)
+
+				if test.expectedVolumeSize != nil {
+					assert.Equal(t, *test.expectedVolumeSize, backupPVC.Spec.Resources.Requests[corev1.ResourceStorage])
+				} else {
+					assert.Equal(t, *resource.NewQuantity(restoreSize, ""), backupPVC.Spec.Resources.Requests[corev1.ResourceStorage])
+				}
 			} else {
 				assert.EqualError(t, err, test.err)
 			}
@@ -384,3 +444,180 @@ func TestExpose(t *testing.T) {
 		})
 	}
 }
+
+func TestGetExpose(t *testing.T) {
+	backup := &velerov1.Backup{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: velerov1.SchemeGroupVersion.String(),
+			Kind:       "Backup",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: velerov1.DefaultNamespace,
+			Name:      "fake-backup",
+			UID:       "fake-uid",
+		},
+	}
+
+	backupPod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: backup.Namespace,
+			Name:      backup.Name,
+		},
+		Spec: corev1.PodSpec{
+			Volumes: []corev1.Volume{
+				{
+					Name: "fake-volume",
+				},
+				{
+					Name: "fake-volume-2",
+				},
+				{
+					Name: string(backup.UID),
+				},
+			},
+		},
+	}
+
+	backupPodWithoutVolume := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: backup.Namespace,
+			Name:      backup.Name,
+		},
+		Spec: corev1.PodSpec{
+			Volumes: []corev1.Volume{
+				{
+					Name: "fake-volume-1",
+				},
+				{
+					Name: "fake-volume-2",
+				},
+			},
+		},
+	}
+
+	backupPVC := &corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: backup.Namespace,
+			Name:      backup.Name,
+		},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			VolumeName: "fake-pv-name",
+		},
+	}
+
+	backupPV := &corev1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "fake-pv-name",
+		},
+	}
+
+	scheme := runtime.NewScheme()
+	corev1.AddToScheme(scheme)
+
+	tests := []struct {
+		name            string
+		kubeClientObj   []runtime.Object
+		ownerBackup     *velerov1.Backup
+		exposeWaitParam CSISnapshotExposeWaitParam
+		Timeout         time.Duration
+		err             string
+		expectedResult  *ExposeResult
+	}{
+		{
+			name:        "backup pod is not found",
+			ownerBackup: backup,
+			exposeWaitParam: CSISnapshotExposeWaitParam{
+				NodeName: "fake-node",
+			},
+		},
+		{
+			name:        "wait pvc bound fail",
+			ownerBackup: backup,
+			exposeWaitParam: CSISnapshotExposeWaitParam{
+				NodeName: "fake-node",
+			},
+			kubeClientObj: []runtime.Object{
+				backupPod,
+			},
+			Timeout: time.Second,
+			err:     "error to wait backup PVC bound, fake-backup: error to wait for rediness of PVC: error to get pvc velero/fake-backup: persistentvolumeclaims \"fake-backup\" not found",
+		},
+		{
+			name:        "backup volume not found in pod",
+			ownerBackup: backup,
+			exposeWaitParam: CSISnapshotExposeWaitParam{
+				NodeName: "fake-node",
+			},
+			kubeClientObj: []runtime.Object{
+				backupPodWithoutVolume,
+				backupPVC,
+				backupPV,
+			},
+			Timeout: time.Second,
+			err:     "backup pod fake-backup doesn't have the expected backup volume",
+		},
+		{
+			name:        "succeed",
+			ownerBackup: backup,
+			exposeWaitParam: CSISnapshotExposeWaitParam{
+				NodeName: "fake-node",
+			},
+			kubeClientObj: []runtime.Object{
+				backupPod,
+				backupPVC,
+				backupPV,
+			},
+			Timeout: time.Second,
+			expectedResult: &ExposeResult{
+				ByPod: ExposeByPod{
+					HostingPod: backupPod,
+					VolumeName: string(backup.UID),
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
+
+			fakeClientBuilder := clientFake.NewClientBuilder()
+			fakeClientBuilder = fakeClientBuilder.WithScheme(scheme)
+
+			fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build()
+
+			exposer := csiSnapshotExposer{
+				kubeClient: fakeKubeClient,
+				log:        velerotest.NewLogger(),
+			}
+
+			var ownerObject corev1.ObjectReference
+			if test.ownerBackup != nil {
+				ownerObject = corev1.ObjectReference{
+					Kind:       test.ownerBackup.Kind,
+					Namespace:  test.ownerBackup.Namespace,
+					Name:       test.ownerBackup.Name,
+					UID:        test.ownerBackup.UID,
+					APIVersion: test.ownerBackup.APIVersion,
+				}
+			}
+
+			test.exposeWaitParam.NodeClient = fakeClient
+
+			result, err := exposer.GetExposed(context.Background(), ownerObject, test.Timeout, &test.exposeWaitParam)
+			if test.err == "" {
+				assert.NoError(t, err)
+
+				if test.expectedResult == nil {
+					assert.Nil(t, result)
+				} else {
+					assert.NoError(t, err)
+					assert.Equal(t, test.expectedResult.ByPod.VolumeName, result.ByPod.VolumeName)
+					assert.Equal(t, test.expectedResult.ByPod.HostingPod.Name, result.ByPod.HostingPod.Name)
+				}
+			} else {
+				assert.EqualError(t, err, test.err)
+			}
+		})
+	}
+}

pkg/features/mocks/PluginFinder.go

@@ -0,0 +1,43 @@
+// Code generated by mockery v2.22.1. DO NOT EDIT.
+
+package mocks
+
+import (
+	common "github.com/vmware-tanzu/velero/pkg/plugin/framework/common"
+
+	mock "github.com/stretchr/testify/mock"
+)
+
+// PluginFinder is an autogenerated mock type for the PluginFinder type
+type PluginFinder struct {
+	mock.Mock
+}
+
+// Find provides a mock function with given fields: kind, name
+func (_m *PluginFinder) Find(kind common.PluginKind, name string) bool {
+	ret := _m.Called(kind, name)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func(common.PluginKind, string) bool); ok {
+		r0 = rf(kind, name)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+type mockConstructorTestingTNewPluginFinder interface {
+	mock.TestingT
+	Cleanup(func())
+}
+
+// NewPluginFinder creates a new instance of PluginFinder. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewPluginFinder(t mockConstructorTestingTNewPluginFinder) *PluginFinder {
+	mock := &PluginFinder{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
+}

pkg/features/mocks/Verifier.go

@@ -0,0 +1,49 @@
+// Code generated by mockery v2.22.1. DO NOT EDIT.
+
+package mocks
+
+import mock "github.com/stretchr/testify/mock"
+
+// Verifier is an autogenerated mock type for the Verifier type
+type Verifier struct {
+	mock.Mock
+}
+
+// Verify provides a mock function with given fields: name
+func (_m *Verifier) Verify(name string) (bool, error) {
+	ret := _m.Called(name)
+
+	var r0 bool
+	var r1 error
+	if rf, ok := ret.Get(0).(func(string) (bool, error)); ok {
+		return rf(name)
+	}
+	if rf, ok := ret.Get(0).(func(string) bool); ok {
+		r0 = rf(name)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(name)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+type mockConstructorTestingTNewVerifier interface {
+	mock.TestingT
+	Cleanup(func())
+}
+
+// NewVerifier creates a new instance of Verifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewVerifier(t mockConstructorTestingTNewVerifier) *Verifier {
+	mock := &Verifier{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
+}

pkg/features/verify.go

@@ -0,0 +1,71 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"errors"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/plugin/framework/common"
+)
+
+type PluginFinder interface {
+	Find(kind common.PluginKind, name string) bool
+}
+
+type Verifier interface {
+	Verify(name string) (bool, error)
+}
+
+type verifier struct {
+	finder PluginFinder
+}
+
+func NewVerifier(finder PluginFinder) Verifier {
+	return &verifier{
+		finder: finder,
+	}
+}
+
+func (v *verifier) Verify(name string) (bool, error) {
+	enabled := IsEnabled(name)
+
+	switch name {
+	case velerov1api.CSIFeatureFlag:
+		return verifyCSIFeature(v.finder, enabled)
+	default:
+		return false, nil
+	}
+}
+
+func verifyCSIFeature(finder PluginFinder, enabled bool) (bool, error) {
+	installed := false
+	installed = finder.Find(common.PluginKindBackupItemActionV2, "velero.io/csi-pvc-backupper")
+	if installed {
+		installed = finder.Find(common.PluginKindRestoreItemActionV2, "velero.io/csi-pvc-restorer")
+	}
+
+	if !enabled && installed {
+		return false, errors.New("CSI plugins are registered, but the EnableCSI feature is not enabled")
+	} else if enabled && !installed {
+		return false, errors.New("CSI feature is enabled, but CSI plugins are not registered")
+	} else if !enabled && !installed {
+		return false, nil
+	} else {
+		return true, nil
+	}
+}

pkg/features/verify_test.go

@@ -0,0 +1,61 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+
+	findermocks "github.com/vmware-tanzu/velero/pkg/features/mocks"
+)
+
+func TestVerify(t *testing.T) {
+	NewFeatureFlagSet()
+	verifier := verifier{}
+
+	finder := new(findermocks.PluginFinder)
+	finder.On("Find", mock.Anything, mock.Anything).Return(false)
+	verifier.finder = finder
+	ready, err := verifier.Verify("EnableCSI")
+	assert.Equal(t, false, ready)
+	assert.Nil(t, err)
+
+	finder = new(findermocks.PluginFinder)
+	finder.On("Find", mock.Anything, mock.Anything).Return(true)
+	verifier.finder = finder
+	ready, err = verifier.Verify("EnableCSI")
+	assert.Equal(t, false, ready)
+	assert.EqualError(t, err, "CSI plugins are registered, but the EnableCSI feature is not enabled")
+
+	Enable("EnableCSI")
+	finder = new(findermocks.PluginFinder)
+	finder.On("Find", mock.Anything, mock.Anything).Return(false)
+	verifier.finder = finder
+	ready, err = verifier.Verify("EnableCSI")
+	assert.Equal(t, false, ready)
+	assert.EqualError(t, err, "CSI feature is enabled, but CSI plugins are not registered")
+
+	Enable("EnableCSI")
+	finder = new(findermocks.PluginFinder)
+	finder.On("Find", mock.Anything, mock.Anything).Return(true)
+	verifier.finder = finder
+	ready, err = verifier.Verify("EnableCSI")
+	assert.Equal(t, true, ready)
+	assert.Nil(t, err)
+}

pkg/nodeagent/node_agent.go

@@ -18,28 +18,52 @@ package nodeagent
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/client-go/kubernetes"
-
-	"github.com/vmware-tanzu/velero/pkg/util/kube"
-
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
 )
 
 const (
 	// daemonSet is the name of the Velero node agent daemonset.
-	daemonSet = "node-agent"
+	daemonSet             = "node-agent"
+	configName            = "node-agent-configs"
+	dataPathConConfigName = "data-path-concurrency"
 )
 
 var (
 	ErrDaemonSetNotFound = errors.New("daemonset not found")
 )
 
+type DataPathConcurrency struct {
+	// GlobalConfig specifies the concurrency number to all nodes for which per-node config is not specified
+	GlobalConfig int `json:"globalConfig,omitempty"`
+
+	// PerNodeConfig specifies the concurrency number to nodes matched by rules
+	PerNodeConfig []RuledConfigs `json:"perNodeConfig,omitempty"`
+}
+
+type RuledConfigs struct {
+	// NodeSelector specifies the label selector to match nodes
+	NodeSelector metav1.LabelSelector `json:"nodeSelector"`
+
+	// Number specifies the number value associated to the matched nodes
+	Number int `json:"number"`
+}
+
+type Configs struct {
+	// DataPathConcurrency is the config for data path concurrency per node.
+	DataPathConcurrency *DataPathConcurrency `json:"dataPathConcurrency,omitempty"`
+}
+
 // IsRunning checks if the node agent daemonset is running properly. If not, return the error found
 func IsRunning(ctx context.Context, kubeClient kubernetes.Interface, namespace string) error {
 	if _, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, daemonSet, metav1.GetOptions{}); apierrors.IsNotFound(err) {
@@ -52,12 +76,18 @@ func IsRunning(ctx context.Context, kubeClient kubernetes.Interface, namespace s
 }
 
 // IsRunningInNode checks if the node agent pod is running properly in a specified node. If not, return the error found
-func IsRunningInNode(ctx context.Context, namespace string, nodeName string, podClient corev1client.PodsGetter) error {
+func IsRunningInNode(ctx context.Context, namespace string, nodeName string, crClient ctrlclient.Client) error {
 	if nodeName == "" {
 		return errors.New("node name is empty")
 	}
 
-	pods, err := podClient.Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("name=%s", daemonSet)})
+	pods := new(v1.PodList)
+	parsedSelector, err := labels.Parse(fmt.Sprintf("name=%s", daemonSet))
+	if err != nil {
+		return errors.Wrap(err, "fail to parse selector")
+	}
+
+	err = crClient.List(ctx, pods, &ctrlclient.ListOptions{LabelSelector: parsedSelector})
 	if err != nil {
 		return errors.Wrap(err, "failed to list daemonset pods")
 	}
@@ -83,3 +113,31 @@ func GetPodSpec(ctx context.Context, kubeClient kubernetes.Interface, namespace
 
 	return &ds.Spec.Template.Spec, nil
 }
+
+func GetConfigs(ctx context.Context, namespace string, kubeClient kubernetes.Interface) (*Configs, error) {
+	cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil, nil
+		} else {
+			return nil, errors.Wrapf(err, "error to get node agent configs %s", configName)
+		}
+	}
+
+	if cm.Data == nil {
+		return nil, errors.Errorf("data is not available in config map %s", configName)
+	}
+
+	jsonString := ""
+	for _, v := range cm.Data {
+		jsonString = v
+	}
+
+	configs := &Configs{}
+	err = json.Unmarshal([]byte(jsonString), configs)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error to unmarshall configs from %s", configName)
+	}
+
+	return configs, nil
+}

pkg/nodeagent/node_agent_test.go

@@ -0,0 +1,337 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodeagent
+
+import (
+	"context"
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/fake"
+	clientTesting "k8s.io/client-go/testing"
+	clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"github.com/vmware-tanzu/velero/pkg/builder"
+)
+
+type reactor struct {
+	verb        string
+	resource    string
+	reactorFunc clientTesting.ReactionFunc
+}
+
+func TestIsRunning(t *testing.T) {
+	daemonSet := &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "fake-ns",
+			Name:      "node-agent",
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind: "DaemonSet",
+		},
+	}
+
+	tests := []struct {
+		name          string
+		kubeClientObj []runtime.Object
+		namespace     string
+		kubeReactors  []reactor
+		expectErr     string
+	}{
+		{
+			name:      "ds is not found",
+			namespace: "fake-ns",
+			expectErr: "daemonset not found",
+		},
+		{
+			name:      "ds get error",
+			namespace: "fake-ns",
+			kubeReactors: []reactor{
+				{
+					verb:     "get",
+					resource: "daemonsets",
+					reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
+						return true, nil, errors.New("fake-get-error")
+					},
+				},
+			},
+			expectErr: "fake-get-error",
+		},
+		{
+			name:      "succeed",
+			namespace: "fake-ns",
+			kubeClientObj: []runtime.Object{
+				daemonSet,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
+
+			for _, reactor := range test.kubeReactors {
+				fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc)
+			}
+
+			err := IsRunning(context.TODO(), fakeKubeClient, test.namespace)
+			if test.expectErr == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.EqualError(t, err, test.expectErr)
+			}
+		})
+	}
+}
+
+func TestIsRunningInNode(t *testing.T) {
+	scheme := runtime.NewScheme()
+	corev1.AddToScheme(scheme)
+
+	nonNodeAgentPod := builder.ForPod("fake-ns", "fake-pod").Result()
+	nodeAgentPodNotRunning := builder.ForPod("fake-ns", "fake-pod").Labels(map[string]string{"name": "node-agent"}).Result()
+	nodeAgentPodRunning1 := builder.ForPod("fake-ns", "fake-pod-1").Labels(map[string]string{"name": "node-agent"}).Phase(corev1.PodRunning).Result()
+	nodeAgentPodRunning2 := builder.ForPod("fake-ns", "fake-pod-2").Labels(map[string]string{"name": "node-agent"}).Phase(corev1.PodRunning).Result()
+	nodeAgentPodRunning3 := builder.ForPod("fake-ns", "fake-pod-3").
+		Labels(map[string]string{"name": "node-agent"}).
+		Phase(corev1.PodRunning).
+		NodeName("fake-node").
+		Result()
+
+	tests := []struct {
+		name          string
+		kubeClientObj []runtime.Object
+		nodeName      string
+		expectErr     string
+	}{
+		{
+			name:      "node name is empty",
+			expectErr: "node name is empty",
+		},
+		{
+			name:     "ds pod not found",
+			nodeName: "fake-node",
+			kubeClientObj: []runtime.Object{
+				nonNodeAgentPod,
+			},
+			expectErr: "daemonset pod not found in running state in node fake-node",
+		},
+		{
+			name:     "ds po are not all running",
+			nodeName: "fake-node",
+			kubeClientObj: []runtime.Object{
+				nodeAgentPodNotRunning,
+				nodeAgentPodRunning1,
+			},
+			expectErr: "daemonset pod not found in running state in node fake-node",
+		},
+		{
+			name:     "ds pods wrong node name",
+			nodeName: "fake-node",
+			kubeClientObj: []runtime.Object{
+				nodeAgentPodNotRunning,
+				nodeAgentPodRunning1,
+				nodeAgentPodRunning2,
+			},
+			expectErr: "daemonset pod not found in running state in node fake-node",
+		},
+		{
+			name:     "succeed",
+			nodeName: "fake-node",
+			kubeClientObj: []runtime.Object{
+				nodeAgentPodNotRunning,
+				nodeAgentPodRunning1,
+				nodeAgentPodRunning2,
+				nodeAgentPodRunning3,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeClientBuilder := clientFake.NewClientBuilder()
+			fakeClientBuilder = fakeClientBuilder.WithScheme(scheme)
+
+			fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build()
+
+			err := IsRunningInNode(context.TODO(), "", test.nodeName, fakeClient)
+			if test.expectErr == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.EqualError(t, err, test.expectErr)
+			}
+		})
+	}
+}
+
+func TestGetPodSpec(t *testing.T) {
+	podSpec := corev1.PodSpec{
+		NodeName: "fake-node",
+	}
+
+	daemonSet := &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "fake-ns",
+			Name:      "node-agent",
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind: "DaemonSet",
+		},
+		Spec: appsv1.DaemonSetSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: podSpec,
+			},
+		},
+	}
+
+	tests := []struct {
+		name          string
+		kubeClientObj []runtime.Object
+		namespace     string
+		expectErr     string
+		expectSpec    corev1.PodSpec
+	}{
+		{
+			name:      "ds is not found",
+			namespace: "fake-ns",
+			expectErr: "error to get node-agent daemonset: daemonsets.apps \"node-agent\" not found",
+		},
+		{
+			name:      "succeed",
+			namespace: "fake-ns",
+			kubeClientObj: []runtime.Object{
+				daemonSet,
+			},
+			expectSpec: podSpec,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
+
+			spec, err := GetPodSpec(context.TODO(), fakeKubeClient, test.namespace)
+			if test.expectErr == "" {
+				assert.NoError(t, err)
+				assert.Equal(t, *spec, test.expectSpec)
+			} else {
+				assert.EqualError(t, err, test.expectErr)
+			}
+		})
+	}
+}
+
+func TestGetConfigs(t *testing.T) {
+	cm := builder.ForConfigMap("fake-ns", "node-agent-configs").Result()
+	cmWithInvalidDataFormat := builder.ForConfigMap("fake-ns", "node-agent-configs").Data("fake-key", "wrong").Result()
+	cmWithoutCocurrentData := builder.ForConfigMap("fake-ns", "node-agent-configs").Data("fake-key", "{\"someothers\":{\"someother\": 10}}").Result()
+	cmWithValidData := builder.ForConfigMap("fake-ns", "node-agent-configs").Data("fake-key", "{\"dataPathConcurrency\":{\"globalConfig\": 5}}").Result()
+
+	tests := []struct {
+		name          string
+		kubeClientObj []runtime.Object
+		namespace     string
+		kubeReactors  []reactor
+		expectResult  *Configs
+		expectErr     string
+	}{
+		{
+			name:      "cm is not found",
+			namespace: "fake-ns",
+		},
+		{
+			name:      "cm get error",
+			namespace: "fake-ns",
+			kubeReactors: []reactor{
+				{
+					verb:     "get",
+					resource: "configmaps",
+					reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
+						return true, nil, errors.New("fake-get-error")
+					},
+				},
+			},
+			expectErr: "error to get node agent configs node-agent-configs: fake-get-error",
+		},
+		{
+			name:      "cm's data is nil",
+			namespace: "fake-ns",
+			kubeClientObj: []runtime.Object{
+				cm,
+			},
+			expectErr: "data is not available in config map node-agent-configs",
+		},
+		{
+			name:      "cm's data is with invalid format",
+			namespace: "fake-ns",
+			kubeClientObj: []runtime.Object{
+				cmWithInvalidDataFormat,
+			},
+			expectErr: "error to unmarshall configs from node-agent-configs: invalid character 'w' looking for beginning of value",
+		},
+		{
+			name:      "concurrency configs are not found",
+			namespace: "fake-ns",
+			kubeClientObj: []runtime.Object{
+				cmWithoutCocurrentData,
+			},
+			expectResult: &Configs{nil},
+		},
+		{
+			name:      "success",
+			namespace: "fake-ns",
+			kubeClientObj: []runtime.Object{
+				cmWithValidData,
+			},
+			expectResult: &Configs{
+				DataPathConcurrency: &DataPathConcurrency{
+					GlobalConfig: 5,
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
+
+			for _, reactor := range test.kubeReactors {
+				fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc)
+			}
+
+			result, err := GetConfigs(context.TODO(), test.namespace, fakeKubeClient)
+			if test.expectErr == "" {
+				assert.NoError(t, err)
+
+				if test.expectResult == nil {
+					assert.Nil(t, result)
+				} else if test.expectResult.DataPathConcurrency == nil {
+					assert.Nil(t, result.DataPathConcurrency)
+				} else {
+					assert.Equal(t, *test.expectResult.DataPathConcurrency, *result.DataPathConcurrency)
+				}
+			} else {
+				assert.EqualError(t, err, test.expectErr)
+			}
+		})
+	}
+}

pkg/persistence/object_store.go

@@ -27,13 +27,14 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
-	"github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/scheme"
 	"github.com/vmware-tanzu/velero/pkg/itemoperation"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
+	"github.com/vmware-tanzu/velero/pkg/util"
 	"github.com/vmware-tanzu/velero/pkg/volume"
 )
 
@@ -49,7 +50,8 @@ type BackupInfo struct {
 	BackupResourceList,
 	CSIVolumeSnapshots,
 	CSIVolumeSnapshotContents,
-	CSIVolumeSnapshotClasses io.Reader
+	CSIVolumeSnapshotClasses,
+	BackupVolumeInfo io.Reader
 }
 
 // BackupStore defines operations for creating, retrieving, and deleting
@@ -269,6 +271,7 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error {
 		s.layout.getCSIVolumeSnapshotContentsKey(info.Name): info.CSIVolumeSnapshotContents,
 		s.layout.getCSIVolumeSnapshotClassesKey(info.Name):  info.CSIVolumeSnapshotClasses,
 		s.layout.getBackupResultsKey(info.Name):             info.BackupResults,
+		s.layout.getBackupVolumeInfoKey(info.Name):          info.BackupVolumeInfo,
 	}
 
 	for key, reader := range backupObjs {
@@ -302,7 +305,9 @@ func (s *objectBackupStore) GetBackupMetadata(name string) (*velerov1api.Backup,
 		return nil, errors.WithStack(err)
 	}
 
-	decoder := scheme.Codecs.UniversalDecoder(velerov1api.SchemeGroupVersion)
+	codecFactory := serializer.NewCodecFactory(util.VeleroScheme)
+
+	decoder := codecFactory.UniversalDecoder(velerov1api.SchemeGroupVersion)
 	obj, _, err := decoder.Decode(data, nil, nil)
 	if err != nil {
 		return nil, errors.WithStack(err)
@@ -488,6 +493,25 @@ func (s *objectBackupStore) GetPodVolumeBackups(name string) ([]*velerov1api.Pod
 	return podVolumeBackups, nil
 }
 
+func (s *objectBackupStore) GetBackupVolumeInfos(name string) (*volume.VolumeInfos, error) {
+	var volumeInfos *volume.VolumeInfos
+
+	res, err := tryGet(s.objectStore, s.bucket, s.layout.getBackupVolumeInfoKey(name))
+	if err != nil {
+		return volumeInfos, err
+	}
+	if res == nil {
+		return volumeInfos, nil
+	}
+	defer res.Close()
+
+	if err := decode(res, &volumeInfos); err != nil {
+		return volumeInfos, err
+	}
+
+	return volumeInfos, nil
+}
+
 func (s *objectBackupStore) GetBackupContents(name string) (io.ReadCloser, error) {
 	return s.objectStore.GetObject(s.bucket, s.layout.getBackupContentsKey(name))
 }

pkg/persistence/object_store_layout.go

@@ -128,3 +128,7 @@ func (l *ObjectStoreLayout) getCSIVolumeSnapshotClassesKey(backup string) string
 func (l *ObjectStoreLayout) getBackupResultsKey(backup string) string {
 	return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-results.gz", backup))
 }
+
+func (l *ObjectStoreLayout) getBackupVolumeInfoKey(backup string) string {
+	return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-volumeinfos.json.gz", backup))
+}

pkg/persistence/object_store_test.go

@@ -227,6 +227,7 @@ func TestPutBackup(t *testing.T) {
 		snapshots            io.Reader
 		backupItemOperations io.Reader
 		resourceList         io.Reader
+		backupVolumeInfo     io.Reader
 		expectedErr          string
 		expectedKeys         []string
 	}{
@@ -239,6 +240,7 @@ func TestPutBackup(t *testing.T) {
 			snapshots:            newStringReadSeeker("snapshots"),
 			backupItemOperations: newStringReadSeeker("backupItemOperations"),
 			resourceList:         newStringReadSeeker("resourceList"),
+			backupVolumeInfo:     newStringReadSeeker("backupVolumeInfo"),
 			expectedErr:          "",
 			expectedKeys: []string{
 				"backups/backup-1/velero-backup.json",
@@ -248,6 +250,7 @@ func TestPutBackup(t *testing.T) {
 				"backups/backup-1/backup-1-volumesnapshots.json.gz",
 				"backups/backup-1/backup-1-itemoperations.json.gz",
 				"backups/backup-1/backup-1-resource-list.json.gz",
+				"backups/backup-1/backup-1-volumeinfos.json.gz",
 			},
 		},
 		{
@@ -260,6 +263,7 @@ func TestPutBackup(t *testing.T) {
 			snapshots:            newStringReadSeeker("snapshots"),
 			backupItemOperations: newStringReadSeeker("backupItemOperations"),
 			resourceList:         newStringReadSeeker("resourceList"),
+			backupVolumeInfo:     newStringReadSeeker("backupVolumeInfo"),
 			expectedErr:          "",
 			expectedKeys: []string{
 				"prefix-1/backups/backup-1/velero-backup.json",
@@ -269,6 +273,7 @@ func TestPutBackup(t *testing.T) {
 				"prefix-1/backups/backup-1/backup-1-volumesnapshots.json.gz",
 				"prefix-1/backups/backup-1/backup-1-itemoperations.json.gz",
 				"prefix-1/backups/backup-1/backup-1-resource-list.json.gz",
+				"prefix-1/backups/backup-1/backup-1-volumeinfos.json.gz",
 			},
 		},
 		{
@@ -280,6 +285,7 @@ func TestPutBackup(t *testing.T) {
 			snapshots:            newStringReadSeeker("snapshots"),
 			backupItemOperations: newStringReadSeeker("backupItemOperations"),
 			resourceList:         newStringReadSeeker("resourceList"),
+			backupVolumeInfo:     newStringReadSeeker("backupVolumeInfo"),
 			expectedErr:          "error readers return errors",
 			expectedKeys:         []string{"backups/backup-1/backup-1-logs.gz"},
 		},
@@ -291,6 +297,7 @@ func TestPutBackup(t *testing.T) {
 			snapshots:            newStringReadSeeker("snapshots"),
 			backupItemOperations: newStringReadSeeker("backupItemOperations"),
 			resourceList:         newStringReadSeeker("resourceList"),
+			backupVolumeInfo:     newStringReadSeeker("backupVolumeInfo"),
 			expectedErr:          "error readers return errors",
 			expectedKeys:         []string{"backups/backup-1/backup-1-logs.gz"},
 		},
@@ -303,6 +310,7 @@ func TestPutBackup(t *testing.T) {
 			snapshots:            newStringReadSeeker("snapshots"),
 			backupItemOperations: newStringReadSeeker("backupItemOperations"),
 			resourceList:         newStringReadSeeker("resourceList"),
+			backupVolumeInfo:     newStringReadSeeker("backupVolumeInfo"),
 			expectedErr:          "",
 			expectedKeys: []string{
 				"backups/backup-1/velero-backup.json",
@@ -311,23 +319,26 @@ func TestPutBackup(t *testing.T) {
 				"backups/backup-1/backup-1-volumesnapshots.json.gz",
 				"backups/backup-1/backup-1-itemoperations.json.gz",
 				"backups/backup-1/backup-1-resource-list.json.gz",
+				"backups/backup-1/backup-1-volumeinfos.json.gz",
 			},
 		},
 		{
-			name:            "data should be uploaded even when metadata is nil",
-			metadata:        nil,
-			contents:        newStringReadSeeker("contents"),
-			log:             newStringReadSeeker("log"),
-			podVolumeBackup: newStringReadSeeker("podVolumeBackup"),
-			snapshots:       newStringReadSeeker("snapshots"),
-			resourceList:    newStringReadSeeker("resourceList"),
-			expectedErr:     "",
+			name:             "data should be uploaded even when metadata is nil",
+			metadata:         nil,
+			contents:         newStringReadSeeker("contents"),
+			log:              newStringReadSeeker("log"),
+			podVolumeBackup:  newStringReadSeeker("podVolumeBackup"),
+			snapshots:        newStringReadSeeker("snapshots"),
+			resourceList:     newStringReadSeeker("resourceList"),
+			backupVolumeInfo: newStringReadSeeker("backupVolumeInfo"),
+			expectedErr:      "",
 			expectedKeys: []string{
 				"backups/backup-1/backup-1.tar.gz",
 				"backups/backup-1/backup-1-logs.gz",
 				"backups/backup-1/backup-1-podvolumebackups.json.gz",
 				"backups/backup-1/backup-1-volumesnapshots.json.gz",
 				"backups/backup-1/backup-1-resource-list.json.gz",
+				"backups/backup-1/backup-1-volumeinfos.json.gz",
 			},
 		},
 	}
@@ -345,6 +356,7 @@ func TestPutBackup(t *testing.T) {
 				VolumeSnapshots:      tc.snapshots,
 				BackupItemOperations: tc.backupItemOperations,
 				BackupResourceList:   tc.resourceList,
+				BackupVolumeInfo:     tc.backupVolumeInfo,
 			}
 			err := harness.PutBackup(backupInfo)
 
@@ -1045,6 +1057,90 @@ func TestNewObjectBackupStoreGetterConfig(t *testing.T) {
 	}
 }
 
+func TestGetBackupVolumeInfos(t *testing.T) {
+	tests := []struct {
+		name           string
+		volumeInfo     *volume.VolumeInfos
+		volumeInfoStr  string
+		expectedErr    string
+		expectedResult []volume.VolumeInfo
+	}{
+		{
+			name: "No VolumeInfos, expect no error.",
+		},
+		{
+			name: "Valid VolumeInfo, should pass.",
+			volumeInfo: &volume.VolumeInfos{
+				VolumeInfos: []volume.VolumeInfo{
+					{
+						PVCName:           "pvcName",
+						PVName:            "pvName",
+						Skipped:           true,
+						SnapshotDataMoved: false,
+					},
+				},
+			},
+			expectedResult: []volume.VolumeInfo{
+				{
+					PVCName:           "pvcName",
+					PVName:            "pvName",
+					Skipped:           true,
+					SnapshotDataMoved: false,
+				},
+			},
+		},
+		{
+			name:          "Invalid VolumeInfo string, should also pass.",
+			volumeInfoStr: `{"volumeInfos": [{"abc": "123", "def": "456", "pvcName": "pvcName"}]}`,
+			expectedResult: []volume.VolumeInfo{
+				{
+					PVCName: "pvcName",
+				},
+			},
+		},
+	}
+
+	harness := newObjectBackupStoreTestHarness("test-bucket", "")
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.volumeInfo != nil {
+				obj := new(bytes.Buffer)
+				gzw := gzip.NewWriter(obj)
+
+				require.NoError(t, json.NewEncoder(gzw).Encode(tc.volumeInfo))
+				require.NoError(t, gzw.Close())
+				harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-volumeinfos.json.gz", obj)
+			}
+
+			if tc.volumeInfoStr != "" {
+				obj := new(bytes.Buffer)
+				gzw := gzip.NewWriter(obj)
+				_, err := gzw.Write([]byte(tc.volumeInfoStr))
+				require.NoError(t, err)
+
+				require.NoError(t, gzw.Close())
+				harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-volumeinfos.json.gz", obj)
+			}
+
+			result, err := harness.GetBackupVolumeInfos("test-backup")
+			if tc.expectedErr != "" {
+				require.Equal(t, tc.expectedErr, err.Error())
+			} else {
+				if err != nil {
+					fmt.Println(err.Error())
+				}
+				require.NoError(t, err)
+			}
+
+			if len(tc.expectedResult) > 0 {
+				require.Equal(t, tc.expectedResult, result.VolumeInfos)
+			}
+
+		})
+	}
+}
+
 func encodeToBytes(obj runtime.Object) []byte {
 	res, err := encode.Encode(obj, "json")
 	if err != nil {

pkg/plugin/clientmgmt/manager_test.go

@@ -61,6 +61,10 @@ func (r *mockRegistry) Get(kind common.PluginKind, name string) (framework.Plugi
 	return id, args.Error(1)
 }
 
+func (r *mockRegistry) Find(kind common.PluginKind, name string) bool {
+	return false
+}
+
 func TestNewManager(t *testing.T) {
 	logger := test.NewLogger()
 	logLevel := logrus.InfoLevel

pkg/plugin/clientmgmt/process/registry.go

@@ -37,6 +37,9 @@ type Registry interface {
 	List(kind common.PluginKind) []framework.PluginIdentifier
 	// Get returns the PluginIdentifier for kind and name.
 	Get(kind common.PluginKind, name string) (framework.PluginIdentifier, error)
+
+	// Find checks if the specified plugin exists in the registry
+	Find(kind common.PluginKind, name string) bool
 }
 
 // KindAndName is a convenience struct that combines a PluginKind and a name.
@@ -125,6 +128,12 @@ func (r *registry) Get(kind common.PluginKind, name string) (framework.PluginIde
 	return p, nil
 }
 
+// Contain if the specified plugin exists in the registry
+func (r *registry) Find(kind common.PluginKind, name string) bool {
+	_, found := r.pluginsByID[KindAndName{Kind: kind, Name: name}]
+	return found
+}
+
 // readPluginsDir recursively reads dir looking for plugins.
 func (r *registry) readPluginsDir(dir string) ([]string, error) {
 	if _, err := r.fs.Stat(dir); err != nil {

pkg/plugin/framework/common/plugin_kinds.go

@@ -41,7 +41,7 @@ const (
 	// PluginKindRestoreItemAction represents a restore item action plugin.
 	PluginKindRestoreItemAction PluginKind = "RestoreItemAction"
 
-	// PluginKindRestoreItemAction represents a v2 restore item action plugin.
+	// PluginKindRestoreItemActionV2 represents a v2 restore item action plugin.
 	PluginKindRestoreItemActionV2 PluginKind = "RestoreItemActionV2"
 
 	// PluginKindDeleteItemAction represents a delete item action plugin.

pkg/podvolume/backupper.go

@@ -26,13 +26,13 @@ import (
 	corev1api "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
+	ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/vmware-tanzu/velero/internal/resourcepolicies"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	veleroclient "github.com/vmware-tanzu/velero/pkg/client"
-	clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
 	"github.com/vmware-tanzu/velero/pkg/label"
 	"github.com/vmware-tanzu/velero/pkg/nodeagent"
 	"github.com/vmware-tanzu/velero/pkg/repository"
@@ -50,10 +50,7 @@ type backupper struct {
 	ctx          context.Context
 	repoLocker   *repository.RepoLocker
 	repoEnsurer  *repository.Ensurer
-	veleroClient clientset.Interface
-	pvcClient    corev1client.PersistentVolumeClaimsGetter
-	pvClient     corev1client.PersistentVolumesGetter
-	podClient    corev1client.PodsGetter
+	crClient     ctrlclient.Client
 	uploaderType string
 
 	results     map[string]chan *velerov1api.PodVolumeBackup
@@ -103,32 +100,31 @@ func newBackupper(
 	ctx context.Context,
 	repoLocker *repository.RepoLocker,
 	repoEnsurer *repository.Ensurer,
-	podVolumeBackupInformer cache.SharedIndexInformer,
-	veleroClient clientset.Interface,
-	pvcClient corev1client.PersistentVolumeClaimsGetter,
-	pvClient corev1client.PersistentVolumesGetter,
-	podClient corev1client.PodsGetter,
+	pvbInformer ctrlcache.Informer,
+	crClient ctrlclient.Client,
 	uploaderType string,
+	backup *velerov1api.Backup,
 	log logrus.FieldLogger,
 ) *backupper {
 	b := &backupper{
 		ctx:          ctx,
 		repoLocker:   repoLocker,
 		repoEnsurer:  repoEnsurer,
-		veleroClient: veleroClient,
-		pvcClient:    pvcClient,
-		pvClient:     pvClient,
-		podClient:    podClient,
+		crClient:     crClient,
 		uploaderType: uploaderType,
 
 		results: make(map[string]chan *velerov1api.PodVolumeBackup),
 	}
 
-	podVolumeBackupInformer.AddEventHandler(
+	pvbInformer.AddEventHandler(
 		cache.ResourceEventHandlerFuncs{
 			UpdateFunc: func(_, obj interface{}) {
 				pvb := obj.(*velerov1api.PodVolumeBackup)
 
+				if pvb.GetLabels()[velerov1api.BackupUIDLabel] != string(backup.UID) {
+					return
+				}
+
 				if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed {
 					b.resultsLock.Lock()
 					defer b.resultsLock.Unlock()
@@ -153,7 +149,8 @@ func resultsKey(ns, name string) string {
 
 func (b *backupper) getMatchAction(resPolicies *resourcepolicies.Policies, pvc *corev1api.PersistentVolumeClaim, volume *corev1api.Volume) (*resourcepolicies.Action, error) {
 	if pvc != nil {
-		pv, err := b.pvClient.PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
+		pv := new(corev1api.PersistentVolume)
+		err := b.crClient.Get(context.TODO(), ctrlclient.ObjectKey{Name: pvc.Spec.VolumeName}, pv)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error getting pv for pvc %s", pvc.Spec.VolumeName)
 		}
@@ -173,7 +170,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.
 	}
 	log.Infof("pod %s/%s has volumes to backup: %v", pod.Namespace, pod.Name, volumesToBackup)
 
-	err := nodeagent.IsRunningInNode(b.ctx, backup.Namespace, pod.Spec.NodeName, b.podClient)
+	err := nodeagent.IsRunningInNode(b.ctx, backup.Namespace, pod.Spec.NodeName, b.crClient)
 	if err != nil {
 		return nil, nil, []error{err}
 	}
@@ -213,7 +210,8 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.
 	for _, podVolume := range pod.Spec.Volumes {
 		podVolumes[podVolume.Name] = podVolume
 		if podVolume.PersistentVolumeClaim != nil {
-			pvc, err := b.pvcClient.PersistentVolumeClaims(pod.Namespace).Get(context.TODO(), podVolume.PersistentVolumeClaim.ClaimName, metav1.GetOptions{})
+			pvc := new(corev1api.PersistentVolumeClaim)
+			err := b.crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: pod.Namespace, Name: podVolume.PersistentVolumeClaim.ClaimName}, pvc)
 			if err != nil {
 				errs = append(errs, errors.Wrap(err, "error getting persistent volume claim for volume"))
 				continue
@@ -263,7 +261,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.
 
 		// hostPath volumes are not supported because they're not mounted into /var/lib/kubelet/pods, so our
 		// daemonset pod has no way to access their data.
-		isHostPath, err := isHostPathVolume(&volume, pvc, b.pvClient.PersistentVolumes())
+		isHostPath, err := isHostPathVolume(&volume, pvc, b.crClient)
 		if err != nil {
 			errs = append(errs, errors.Wrap(err, "error checking if volume is a hostPath volume"))
 			continue
@@ -303,11 +301,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.
 		}
 
 		volumeBackup := newPodVolumeBackup(backup, pod, volume, repoIdentifier, b.uploaderType, pvc)
-		// TODO: once backupper is refactored to use controller-runtime, just pass client instead of anonymous func
-		if err := veleroclient.CreateRetryGenerateNameWithFunc(volumeBackup, func() error {
-			_, err := b.veleroClient.VeleroV1().PodVolumeBackups(volumeBackup.Namespace).Create(context.TODO(), volumeBackup, metav1.CreateOptions{})
-			return err
-		}); err != nil {
+		if err := veleroclient.CreateRetryGenerateName(b.crClient, b.ctx, volumeBackup); err != nil {
 			errs = append(errs, err)
 			continue
 		}
@@ -339,13 +333,9 @@ ForEachVolume:
 	return podVolumeBackups, pvcSummary, errs
 }
 
-type pvGetter interface {
-	Get(ctx context.Context, name string, opts metav1.GetOptions) (*corev1api.PersistentVolume, error)
-}
-
 // isHostPathVolume returns true if the volume is either a hostPath pod volume or a persistent
 // volume claim on a hostPath persistent volume, or false otherwise.
-func isHostPathVolume(volume *corev1api.Volume, pvc *corev1api.PersistentVolumeClaim, pvGetter pvGetter) (bool, error) {
+func isHostPathVolume(volume *corev1api.Volume, pvc *corev1api.PersistentVolumeClaim, crClient ctrlclient.Client) (bool, error) {
 	if volume.HostPath != nil {
 		return true, nil
 	}
@@ -354,7 +344,8 @@ func isHostPathVolume(volume *corev1api.Volume, pvc *corev1api.PersistentVolumeC
 		return false, nil
 	}
 
-	pv, err := pvGetter.Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
+	pv := new(corev1api.PersistentVolume)
+	err := crClient.Get(context.TODO(), ctrlclient.ObjectKey{Name: pvc.Spec.VolumeName}, pv)
 	if err != nil {
 		return false, errors.WithStack(err)
 	}
@@ -422,7 +413,3 @@ func newPodVolumeBackup(backup *velerov1api.Backup, pod *corev1api.Pod, volume c
 
 	return pvb
 }
-
-func errorOnly(_ interface{}, err error) error {
-	return err
-}

pkg/podvolume/backupper_factory.go

@@ -18,17 +18,14 @@ package podvolume
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
+	ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
-	clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
-	velerov1informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 )
 
@@ -41,48 +38,31 @@ type BackupperFactory interface {
 func NewBackupperFactory(
 	repoLocker *repository.RepoLocker,
 	repoEnsurer *repository.Ensurer,
-	veleroClient clientset.Interface,
-	pvcClient corev1client.PersistentVolumeClaimsGetter,
-	pvClient corev1client.PersistentVolumesGetter,
-	podClient corev1client.PodsGetter,
+	crClient ctrlclient.Client,
+	pvbInformer ctrlcache.Informer,
 	log logrus.FieldLogger,
 ) BackupperFactory {
 	return &backupperFactory{
-		repoLocker:   repoLocker,
-		repoEnsurer:  repoEnsurer,
-		veleroClient: veleroClient,
-		pvcClient:    pvcClient,
-		pvClient:     pvClient,
-		podClient:    podClient,
-		log:          log,
+		repoLocker:  repoLocker,
+		repoEnsurer: repoEnsurer,
+		crClient:    crClient,
+		pvbInformer: pvbInformer,
+		log:         log,
 	}
 }
 
 type backupperFactory struct {
-	repoLocker   *repository.RepoLocker
-	repoEnsurer  *repository.Ensurer
-	veleroClient clientset.Interface
-	pvcClient    corev1client.PersistentVolumeClaimsGetter
-	pvClient     corev1client.PersistentVolumesGetter
-	podClient    corev1client.PodsGetter
-	log          logrus.FieldLogger
+	repoLocker  *repository.RepoLocker
+	repoEnsurer *repository.Ensurer
+	crClient    ctrlclient.Client
+	pvbInformer ctrlcache.Informer
+	log         logrus.FieldLogger
 }
 
 func (bf *backupperFactory) NewBackupper(ctx context.Context, backup *velerov1api.Backup, uploaderType string) (Backupper, error) {
-	informer := velerov1informers.NewFilteredPodVolumeBackupInformer(
-		bf.veleroClient,
-		backup.Namespace,
-		0,
-		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
-		func(opts *metav1.ListOptions) {
-			opts.LabelSelector = fmt.Sprintf("%s=%s", velerov1api.BackupUIDLabel, backup.UID)
-		},
-	)
+	b := newBackupper(ctx, bf.repoLocker, bf.repoEnsurer, bf.pvbInformer, bf.crClient, uploaderType, backup, bf.log)
 
-	b := newBackupper(ctx, bf.repoLocker, bf.repoEnsurer, informer, bf.veleroClient, bf.pvcClient, bf.pvClient, bf.podClient, uploaderType, bf.log)
-
-	go informer.Run(ctx.Done())
-	if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) {
+	if !cache.WaitForCacheSync(ctx.Done(), bf.pvbInformer.HasSynced) {
 		return nil, errors.New("timed out waiting for caches to sync")
 	}
 

pkg/podvolume/backupper_test.go

@@ -23,7 +23,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,17 +31,15 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
 
-	"k8s.io/client-go/kubernetes"
-	kubefake "k8s.io/client-go/kubernetes/fake"
 	clientTesting "k8s.io/client-go/testing"
+	"k8s.io/client-go/tools/cache"
 
 	"github.com/vmware-tanzu/velero/internal/resourcepolicies"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
-	"github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
-	velerofake "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
 )
 
 func TestIsHostPathVolume(t *testing.T) {
@@ -101,15 +98,14 @@ func TestIsHostPathVolume(t *testing.T) {
 			VolumeName: "pv-1",
 		},
 	}
-	pvGetter := &fakePVGetter{
-		pv: &corev1api.PersistentVolume{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: "pv-1",
-			},
-			Spec: corev1api.PersistentVolumeSpec{},
+	pv := &corev1api.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "pv-1",
 		},
+		Spec: corev1api.PersistentVolumeSpec{},
 	}
-	isHostPath, err = isHostPathVolume(vol, pvc, pvGetter)
+	crClient1 := velerotest.NewFakeControllerRuntimeClient(t, pv)
+	isHostPath, err = isHostPathVolume(vol, pvc, crClient1)
 	assert.Nil(t, err)
 	assert.False(t, isHostPath)
 
@@ -130,35 +126,23 @@ func TestIsHostPathVolume(t *testing.T) {
 			VolumeName: "pv-1",
 		},
 	}
-	pvGetter = &fakePVGetter{
-		pv: &corev1api.PersistentVolume{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: "pv-1",
-			},
-			Spec: corev1api.PersistentVolumeSpec{
-				PersistentVolumeSource: corev1api.PersistentVolumeSource{
-					HostPath: &corev1api.HostPathVolumeSource{},
-				},
+	pv = &corev1api.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "pv-1",
+		},
+		Spec: corev1api.PersistentVolumeSpec{
+			PersistentVolumeSource: corev1api.PersistentVolumeSource{
+				HostPath: &corev1api.HostPathVolumeSource{},
 			},
 		},
 	}
-	isHostPath, err = isHostPathVolume(vol, pvc, pvGetter)
+	crClient2 := velerotest.NewFakeControllerRuntimeClient(t, pv)
+
+	isHostPath, err = isHostPathVolume(vol, pvc, crClient2)
 	assert.Nil(t, err)
 	assert.True(t, isHostPath)
 }
 
-type fakePVGetter struct {
-	pv *corev1api.PersistentVolume
-}
-
-func (g *fakePVGetter) Get(ctx context.Context, name string, opts metav1.GetOptions) (*corev1api.PersistentVolume, error) {
-	if g.pv != nil {
-		return g.pv, nil
-	}
-
-	return nil, errors.New("item not found")
-}
-
 func Test_backupper_BackupPodVolumes_log_test(t *testing.T) {
 	type args struct {
 		backup          *velerov1api.Backup
@@ -322,6 +306,7 @@ func createPVBObj(fail bool, withSnapshot bool, index int, uploaderType string)
 func TestBackupPodVolumes(t *testing.T) {
 	scheme := runtime.NewScheme()
 	velerov1api.AddToScheme(scheme)
+	corev1api.AddToScheme(scheme)
 
 	ctxWithCancel, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -509,40 +494,6 @@ func TestBackupPodVolumes(t *testing.T) {
 			uploaderType:  "kopia",
 			bsl:           "fake-bsl",
 		},
-		{
-			name: "create PVB fail",
-			volumes: []string{
-				"fake-volume-1",
-				"fake-volume-2",
-			},
-			sourcePod: createPodObj(true, true, true, 2),
-			kubeClientObj: []runtime.Object{
-				createNodeAgentPodObj(true),
-				createPVCObj(1),
-				createPVCObj(2),
-				createPVObj(1, false),
-				createPVObj(2, false),
-			},
-			ctlClientObj: []runtime.Object{
-				createBackupRepoObj(),
-			},
-			veleroReactors: []reactor{
-				{
-					verb:     "create",
-					resource: "podvolumebackups",
-					reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
-						return true, nil, errors.New("fake-create-error")
-					},
-				},
-			},
-			runtimeScheme: scheme,
-			uploaderType:  "kopia",
-			bsl:           "fake-bsl",
-			errs: []string{
-				"fake-create-error",
-				"fake-create-error",
-			},
-		},
 		{
 			name: "context cancelled",
 			ctx:  ctxWithCancel,
@@ -630,23 +581,28 @@ func TestBackupPodVolumes(t *testing.T) {
 				fakeClientBuilder = fakeClientBuilder.WithScheme(test.runtimeScheme)
 			}
 
-			fakeCtlClient := fakeClientBuilder.WithRuntimeObjects(test.ctlClientObj...).Build()
+			objList := append(test.ctlClientObj, test.veleroClientObj...)
+			objList = append(objList, test.kubeClientObj...)
+			fakeCtrlClient := fakeClientBuilder.WithRuntimeObjects(objList...).Build()
 
-			fakeKubeClient := kubefake.NewSimpleClientset(test.kubeClientObj...)
-			var kubeClient kubernetes.Interface = fakeKubeClient
-
-			fakeVeleroClient := velerofake.NewSimpleClientset(test.veleroClientObj...)
-			for _, reactor := range test.veleroReactors {
-				fakeVeleroClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc)
+			fakeCRWatchClient := velerotest.NewFakeControllerRuntimeWatchClient(t, test.kubeClientObj...)
+			lw := kube.InternalLW{
+				Client:     fakeCRWatchClient,
+				Namespace:  velerov1api.DefaultNamespace,
+				ObjectList: new(velerov1api.PodVolumeBackupList),
 			}
-			var veleroClient versioned.Interface = fakeVeleroClient
 
-			ensurer := repository.NewEnsurer(fakeCtlClient, velerotest.NewLogger(), time.Millisecond)
+			pvbInformer := cache.NewSharedIndexInformer(&lw, &velerov1api.PodVolumeBackup{}, 0, cache.Indexers{})
+
+			go pvbInformer.Run(ctx.Done())
+			require.True(t, cache.WaitForCacheSync(ctx.Done(), pvbInformer.HasSynced))
+
+			ensurer := repository.NewEnsurer(fakeCtrlClient, velerotest.NewLogger(), time.Millisecond)
 
 			backupObj := builder.ForBackup(velerov1api.DefaultNamespace, "fake-backup").Result()
 			backupObj.Spec.StorageLocation = test.bsl
 
-			factory := NewBackupperFactory(repository.NewRepoLocker(), ensurer, veleroClient, kubeClient.CoreV1(), kubeClient.CoreV1(), kubeClient.CoreV1(), velerotest.NewLogger())
+			factory := NewBackupperFactory(repository.NewRepoLocker(), ensurer, fakeCtrlClient, pvbInformer, velerotest.NewLogger())
 			bp, err := factory.NewBackupper(ctx, backupObj, test.uploaderType)
 
 			require.NoError(t, err)

pkg/podvolume/restorer.go

@@ -27,12 +27,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
+	ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	veleroclient "github.com/vmware-tanzu/velero/pkg/client"
-	clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
 	"github.com/vmware-tanzu/velero/pkg/label"
 	"github.com/vmware-tanzu/velero/pkg/nodeagent"
 	"github.com/vmware-tanzu/velero/pkg/repository"
@@ -54,13 +54,11 @@ type Restorer interface {
 }
 
 type restorer struct {
-	ctx          context.Context
-	repoLocker   *repository.RepoLocker
-	repoEnsurer  *repository.Ensurer
-	veleroClient clientset.Interface
-	pvcClient    corev1client.PersistentVolumeClaimsGetter
-	podClient    corev1client.PodsGetter
-	kubeClient   kubernetes.Interface
+	ctx         context.Context
+	repoLocker  *repository.RepoLocker
+	repoEnsurer *repository.Ensurer
+	kubeClient  kubernetes.Interface
+	crClient    ctrlclient.Client
 
 	resultsLock    sync.Mutex
 	results        map[string]chan *velerov1api.PodVolumeRestore
@@ -72,30 +70,30 @@ func newRestorer(
 	ctx context.Context,
 	repoLocker *repository.RepoLocker,
 	repoEnsurer *repository.Ensurer,
-	podVolumeRestoreInformer cache.SharedIndexInformer,
-	veleroClient clientset.Interface,
-	pvcClient corev1client.PersistentVolumeClaimsGetter,
-	podClient corev1client.PodsGetter,
+	pvrInformer ctrlcache.Informer,
 	kubeClient kubernetes.Interface,
+	crClient ctrlclient.Client,
+	restore *velerov1api.Restore,
 	log logrus.FieldLogger,
 ) *restorer {
 	r := &restorer{
-		ctx:          ctx,
-		repoLocker:   repoLocker,
-		repoEnsurer:  repoEnsurer,
-		veleroClient: veleroClient,
-		pvcClient:    pvcClient,
-		podClient:    podClient,
-		kubeClient:   kubeClient,
+		ctx:         ctx,
+		repoLocker:  repoLocker,
+		repoEnsurer: repoEnsurer,
+		kubeClient:  kubeClient,
+		crClient:    crClient,
 
 		results: make(map[string]chan *velerov1api.PodVolumeRestore),
 		log:     log,
 	}
 
-	podVolumeRestoreInformer.AddEventHandler(
+	pvrInformer.AddEventHandler(
 		cache.ResourceEventHandlerFuncs{
 			UpdateFunc: func(_, obj interface{}) {
 				pvr := obj.(*velerov1api.PodVolumeRestore)
+				if pvr.GetLabels()[velerov1api.RestoreUIDLabel] != string(restore.UID) {
+					return
+				}
 
 				if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed {
 					r.resultsLock.Lock()
@@ -169,7 +167,8 @@ func (r *restorer) RestorePodVolumes(data RestoreData) []error {
 		var pvc *corev1api.PersistentVolumeClaim
 		if ok {
 			if volumeObj.PersistentVolumeClaim != nil {
-				pvc, err = r.pvcClient.PersistentVolumeClaims(data.Pod.Namespace).Get(context.TODO(), volumeObj.PersistentVolumeClaim.ClaimName, metav1.GetOptions{})
+				pvc := new(corev1api.PersistentVolumeClaim)
+				err := r.crClient.Get(context.TODO(), ctrlclient.ObjectKey{Namespace: data.Pod.Namespace, Name: volumeObj.PersistentVolumeClaim.ClaimName}, pvc)
 				if err != nil {
 					errs = append(errs, errors.Wrap(err, "error getting persistent volume claim for volume"))
 					continue
@@ -179,10 +178,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData) []error {
 
 		volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc)
 
-		// TODO: once restorer is refactored to use controller-runtime, just pass client instead of anonymous func
-		if err := veleroclient.CreateRetryGenerateNameWithFunc(volumeRestore, func() error {
-			return errorOnly(r.veleroClient.VeleroV1().PodVolumeRestores(volumeRestore.Namespace).Create(context.TODO(), volumeRestore, metav1.CreateOptions{}))
-		}); err != nil {
+		if err := veleroclient.CreateRetryGenerateName(r.crClient, r.ctx, volumeRestore); err != nil {
 			errs = append(errs, errors.WithStack(err))
 			continue
 		}
@@ -214,7 +210,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData) []error {
 		} else if err != nil {
 			r.log.WithError(err).Error("Failed to check node-agent pod status, disengage")
 		} else {
-			err = nodeagent.IsRunningInNode(checkCtx, data.Restore.Namespace, nodeName, r.podClient)
+			err = nodeagent.IsRunningInNode(checkCtx, data.Restore.Namespace, nodeName, r.crClient)
 			if err != nil {
 				r.log.WithField("node", nodeName).WithError(err).Error("node-agent pod is not running in node, abort the restore")
 				r.nodeAgentCheck <- errors.Wrapf(err, "node-agent pod is not running in node %s", nodeName)

pkg/podvolume/restorer_factory.go

@@ -18,18 +18,15 @@ package podvolume
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
+	ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
-	clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
-	velerov1informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 )
 
@@ -41,47 +38,33 @@ type RestorerFactory interface {
 
 func NewRestorerFactory(repoLocker *repository.RepoLocker,
 	repoEnsurer *repository.Ensurer,
-	veleroClient clientset.Interface,
-	pvcClient corev1client.PersistentVolumeClaimsGetter,
-	podClient corev1client.PodsGetter,
 	kubeClient kubernetes.Interface,
+	crClient ctrlclient.Client,
+	pvrInformer ctrlcache.Informer,
 	log logrus.FieldLogger) RestorerFactory {
 	return &restorerFactory{
-		repoLocker:   repoLocker,
-		repoEnsurer:  repoEnsurer,
-		veleroClient: veleroClient,
-		pvcClient:    pvcClient,
-		podClient:    podClient,
-		kubeClient:   kubeClient,
-		log:          log,
+		repoLocker:  repoLocker,
+		repoEnsurer: repoEnsurer,
+		kubeClient:  kubeClient,
+		crClient:    crClient,
+		pvrInformer: pvrInformer,
+		log:         log,
 	}
 }
 
 type restorerFactory struct {
-	repoLocker   *repository.RepoLocker
-	repoEnsurer  *repository.Ensurer
-	veleroClient clientset.Interface
-	pvcClient    corev1client.PersistentVolumeClaimsGetter
-	podClient    corev1client.PodsGetter
-	kubeClient   kubernetes.Interface
-	log          logrus.FieldLogger
+	repoLocker  *repository.RepoLocker
+	repoEnsurer *repository.Ensurer
+	kubeClient  kubernetes.Interface
+	crClient    ctrlclient.Client
+	pvrInformer ctrlcache.Informer
+	log         logrus.FieldLogger
 }
 
 func (rf *restorerFactory) NewRestorer(ctx context.Context, restore *velerov1api.Restore) (Restorer, error) {
-	informer := velerov1informers.NewFilteredPodVolumeRestoreInformer(
-		rf.veleroClient,
-		restore.Namespace,
-		0,
-		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
-		func(opts *metav1.ListOptions) {
-			opts.LabelSelector = fmt.Sprintf("%s=%s", velerov1api.RestoreUIDLabel, restore.UID)
-		},
-	)
+	r := newRestorer(ctx, rf.repoLocker, rf.repoEnsurer, rf.pvrInformer, rf.kubeClient, rf.crClient, restore, rf.log)
 
-	r := newRestorer(ctx, rf.repoLocker, rf.repoEnsurer, informer, rf.veleroClient, rf.pvcClient, rf.podClient, rf.kubeClient, rf.log)
-
-	go informer.Run(ctx.Done())
-	if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) {
+	if !cache.WaitForCacheSync(ctx.Done(), rf.pvrInformer.HasSynced) {
 		return nil, errors.New("timed out waiting for cache to sync")
 	}
 

pkg/podvolume/restorer_test.go

@@ -22,7 +22,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	appv1 "k8s.io/api/apps/v1"
@@ -31,15 +30,14 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	kubefake "k8s.io/client-go/kubernetes/fake"
-	clientTesting "k8s.io/client-go/testing"
+	"k8s.io/client-go/tools/cache"
 	ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
-	"github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned"
-	velerofake "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
 )
 
 func TestGetVolumesRepositoryType(t *testing.T) {
@@ -162,6 +160,7 @@ type expectError struct {
 func TestRestorePodVolumes(t *testing.T) {
 	scheme := runtime.NewScheme()
 	velerov1api.AddToScheme(scheme)
+	corev1api.AddToScheme(scheme)
 
 	ctxWithCancel, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -264,42 +263,6 @@ func TestRestorePodVolumes(t *testing.T) {
 				},
 			},
 		},
-		{
-			name: "create pvb fail",
-			pvbs: []*velerov1api.PodVolumeBackup{
-				createPVBObj(true, true, 1, "kopia"),
-				createPVBObj(true, true, 2, "kopia"),
-			},
-			kubeClientObj: []runtime.Object{
-				createNodeAgentDaemonset(),
-				createPVCObj(1),
-				createPVCObj(2),
-			},
-			ctlClientObj: []runtime.Object{
-				createBackupRepoObj(),
-			},
-			veleroReactors: []reactor{
-				{
-					verb:     "create",
-					resource: "podvolumerestores",
-					reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) {
-						return true, nil, errors.New("fake-create-error")
-					},
-				},
-			},
-			restoredPod:     createPodObj(true, true, true, 2),
-			sourceNamespace: "fake-ns",
-			bsl:             "fake-bsl",
-			runtimeScheme:   scheme,
-			errs: []expectError{
-				{
-					err: "fake-create-error",
-				},
-				{
-					err: "fake-create-error",
-				},
-			},
-		},
 		{
 			name: "create pvb fail",
 			ctx:  ctxWithCancel,
@@ -407,22 +370,32 @@ func TestRestorePodVolumes(t *testing.T) {
 				fakeClientBuilder = fakeClientBuilder.WithScheme(test.runtimeScheme)
 			}
 
-			fakeCtlClient := fakeClientBuilder.WithRuntimeObjects(test.ctlClientObj...).Build()
+			objClient := append(test.ctlClientObj, test.kubeClientObj...)
+			objClient = append(objClient, test.veleroClientObj...)
+
+			fakeCRClient := velerotest.NewFakeControllerRuntimeClient(t, objClient...)
 
 			fakeKubeClient := kubefake.NewSimpleClientset(test.kubeClientObj...)
 			var kubeClient kubernetes.Interface = fakeKubeClient
 
-			fakeVeleroClient := velerofake.NewSimpleClientset(test.veleroClientObj...)
-			for _, reactor := range test.veleroReactors {
-				fakeVeleroClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc)
+			fakeCRWatchClient := velerotest.NewFakeControllerRuntimeWatchClient(t, test.kubeClientObj...)
+			lw := kube.InternalLW{
+				Client:     fakeCRWatchClient,
+				Namespace:  velerov1api.DefaultNamespace,
+				ObjectList: new(velerov1api.PodVolumeRestoreList),
 			}
-			var veleroClient versioned.Interface = fakeVeleroClient
 
-			ensurer := repository.NewEnsurer(fakeCtlClient, velerotest.NewLogger(), time.Millisecond)
+			pvrInformer := cache.NewSharedIndexInformer(&lw, &velerov1api.PodVolumeBackup{}, 0, cache.Indexers{})
+
+			go pvrInformer.Run(ctx.Done())
+			require.True(t, cache.WaitForCacheSync(ctx.Done(), pvrInformer.HasSynced))
+
+			ensurer := repository.NewEnsurer(fakeCRClient, velerotest.NewLogger(), time.Millisecond)
 
 			restoreObj := builder.ForRestore(velerov1api.DefaultNamespace, "fake-restore").Result()
 
-			factory := NewRestorerFactory(repository.NewRepoLocker(), ensurer, veleroClient, kubeClient.CoreV1(), kubeClient.CoreV1(), kubeClient, velerotest.NewLogger())
+			factory := NewRestorerFactory(repository.NewRepoLocker(), ensurer, kubeClient,
+				fakeCRClient, pvrInformer, velerotest.NewLogger())
 			rs, err := factory.NewRestorer(ctx, restoreObj)
 
 			require.NoError(t, err)

pkg/repository/config/aws.go

@@ -21,12 +21,10 @@ import (
 	"context"
 	"os"
 
-	goerr "errors"
-
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/endpoints"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/pkg/errors"
 )
 
@@ -64,7 +62,7 @@ func GetS3ResticEnvVars(config map[string]string) (map[string]string, error) {
 		result[awsSecretKeyEnvVar] = creds.SecretAccessKey
 		result[awsSessTokenEnvVar] = creds.SessionToken
 		result[awsCredentialsFileEnvVar] = ""
-		result[awsProfileEnvVar] = ""
+		result[awsProfileEnvVar] = "" // profile is not needed since we have the credentials from profile via GetS3Credentials
 		result[awsConfigFileEnvVar] = ""
 	}
 
@@ -73,27 +71,29 @@ func GetS3ResticEnvVars(config map[string]string) (map[string]string, error) {
 
 // GetS3Credentials gets the S3 credential values according to the information
 // of the provided config or the system's environment variables
-func GetS3Credentials(config map[string]string) (*credentials.Value, error) {
+func GetS3Credentials(config map[string]string) (*aws.Credentials, error) {
 	if os.Getenv(awsRoleEnvVar) != "" {
 		return nil, nil
 	}
 
-	opts := session.Options{}
+	var opts []func(*awsconfig.LoadOptions) error
 	credentialsFile := config[CredentialsFileKey]
 	if credentialsFile == "" {
 		credentialsFile = os.Getenv(awsCredentialsFileEnvVar)
 	}
 	if credentialsFile != "" {
-		opts.SharedConfigFiles = append(opts.SharedConfigFiles, credentialsFile)
-		opts.SharedConfigState = session.SharedConfigEnable
+		opts = append(opts, awsconfig.WithSharedCredentialsFiles([]string{credentialsFile}),
+			// To support the existing use case where config file is passed
+			// as credentials of a BSL
+			awsconfig.WithSharedConfigFiles([]string{credentialsFile}))
 	}
+	opts = append(opts, awsconfig.WithSharedConfigProfile(config[awsProfileKey]))
 
-	sess, err := session.NewSessionWithOptions(opts)
+	cfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
 	if err != nil {
 		return nil, err
 	}
-
-	creds, err := sess.Config.Credentials.Get()
+	creds, err := cfg.Credentials.Retrieve(context.Background())
 
 	return &creds, err
 }
@@ -101,33 +101,17 @@ func GetS3Credentials(config map[string]string) (*credentials.Value, error) {
 // GetAWSBucketRegion returns the AWS region that a bucket is in, or an error
 // if the region cannot be determined.
 func GetAWSBucketRegion(bucket string) (string, error) {
-	sess, err := session.NewSession()
+	cfg, err := awsconfig.LoadDefaultConfig(context.Background())
 	if err != nil {
 		return "", errors.WithStack(err)
 	}
-
-	var region string
-	var requestErrs []error
-
-	for _, partition := range endpoints.DefaultPartitions() {
-		for regionHint := range partition.Regions() {
-			region, err = s3manager.GetBucketRegion(context.Background(), sess, bucket, regionHint)
-			if err != nil {
-				requestErrs = append(requestErrs, errors.Wrapf(err, "error to get region with hint %s", regionHint))
-			}
-
-			// we only need to try a single region hint per partition, so break after the first
-			break
-		}
-
-		if region != "" {
-			return region, nil
-		}
+	client := s3.NewFromConfig(cfg)
+	region, err := s3manager.GetBucketRegion(context.Background(), client, bucket)
+	if err != nil {
+		return "", errors.WithStack(err)
 	}
-
-	if requestErrs == nil {
-		return "", errors.Errorf("unable to determine region by bucket %s", bucket)
-	} else {
-		return "", errors.Wrapf(goerr.Join(requestErrs...), "error to get region by bucket %s", bucket)
+	if region == "" {
+		return "", errors.New("unable to determine bucket's region")
 	}
+	return region, nil
 }

pkg/repository/config/aws_test.go

@@ -17,8 +17,11 @@ limitations under the License.
 package config
 
 import (
+	"os"
+	"reflect"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/stretchr/testify/require"
 )
 
@@ -63,3 +66,81 @@ func TestGetS3ResticEnvVars(t *testing.T) {
 		})
 	}
 }
+
+func TestGetS3CredentialsCorrectlyUseProfile(t *testing.T) {
+	type args struct {
+		config             map[string]string
+		secretFileContents string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *aws.Credentials
+		wantErr bool
+	}{
+		{
+			name: "Test GetS3Credentials use profile correctly",
+			args: args{
+				config: map[string]string{
+					"profile": "some-profile",
+				},
+				secretFileContents: `[default]
+	aws_access_key_id = default-access-key-id
+	aws_secret_access_key = default-secret-access-key
+	[profile some-profile]
+	aws_access_key_id = some-profile-access-key-id
+	aws_secret_access_key = some-profile-secret-access-key
+	`,
+			},
+			want: &aws.Credentials{
+				AccessKeyID:     "some-profile-access-key-id",
+				SecretAccessKey: "some-profile-secret-access-key",
+			},
+		},
+		{
+			name: "Test GetS3Credentials default to default profile",
+			args: args{
+				config: map[string]string{},
+				secretFileContents: `[default]
+	aws_access_key_id = default-access-key-id
+	aws_secret_access_key = default-secret-access-key
+	[profile some-profile]
+	aws_access_key_id = some-profile-access-key-id
+	aws_secret_access_key = some-profile-secret-access-key
+	`,
+			},
+			want: &aws.Credentials{
+				AccessKeyID:     "default-access-key-id",
+				SecretAccessKey: "default-secret-access-key",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tmpFile, err := os.CreateTemp("", "velero-test-aws-credentials")
+			defer os.Remove(tmpFile.Name())
+			if err != nil {
+				t.Errorf("GetS3Credentials() error = %v", err)
+				return
+			}
+			// write the contents of the secret file to the temp file
+			_, err = tmpFile.WriteString(tt.args.secretFileContents)
+			if err != nil {
+				t.Errorf("GetS3Credentials() error = %v", err)
+				return
+			}
+			tt.args.config["credentialsFile"] = tmpFile.Name()
+			got, err := GetS3Credentials(tt.args.config)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetS3Credentials() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got.AccessKeyID, tt.want.AccessKeyID) {
+				t.Errorf("GetS3Credentials() got = %v, want %v", got.AccessKeyID, tt.want.AccessKeyID)
+			}
+			if !reflect.DeepEqual(got.SecretAccessKey, tt.want.SecretAccessKey) {
+				t.Errorf("GetS3Credentials() got = %v, want %v", got.SecretAccessKey, tt.want.SecretAccessKey)
+			}
+		})
+	}
+}

pkg/repository/provider/unified_repo.go

@@ -53,7 +53,7 @@ var getGCPCredentials = repoconfig.GetGCPCredentials
 var getS3BucketRegion = repoconfig.GetAWSBucketRegion
 
 type localFuncTable struct {
-	getStorageVariables   func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error)
+	getStorageVariables   func(*velerov1api.BackupStorageLocation, string, string, credentials.FileStore) (map[string]string, error)
 	getStorageCredentials func(*velerov1api.BackupStorageLocation, credentials.FileStore) (map[string]string, error)
 }
 
@@ -347,7 +347,7 @@ func (urp *unifiedRepoProvider) GetStoreOptions(param interface{}) (map[string]s
 		return map[string]string{}, errors.Errorf("invalid parameter, expect %T, actual %T", RepoParam{}, param)
 	}
 
-	storeVar, err := funcTable.getStorageVariables(repoParam.BackupLocation, urp.repoBackend, repoParam.BackupRepo.Spec.VolumeNamespace)
+	storeVar, err := funcTable.getStorageVariables(repoParam.BackupLocation, urp.repoBackend, repoParam.BackupRepo.Spec.VolumeNamespace, urp.credentialGetter.FromFile)
 	if err != nil {
 		return map[string]string{}, errors.Wrap(err, "error to get storage variables")
 	}
@@ -433,7 +433,7 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr
 
 		if credValue != nil {
 			result[udmrepo.StoreOptionS3KeyID] = credValue.AccessKeyID
-			result[udmrepo.StoreOptionS3Provider] = credValue.ProviderName
+			result[udmrepo.StoreOptionS3Provider] = credValue.Source
 			result[udmrepo.StoreOptionS3SecretKey] = credValue.SecretAccessKey
 			result[udmrepo.StoreOptionS3Token] = credValue.SessionToken
 		}
@@ -447,7 +447,8 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr
 	return result, nil
 }
 
-func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repoBackend string, repoName string) (map[string]string, error) {
+func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repoBackend string, repoName string,
+	credentialFileStore credentials.FileStore) (map[string]string, error) {
 	result := make(map[string]string)
 
 	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config)
@@ -459,6 +460,13 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo
 	if config == nil {
 		config = map[string]string{}
 	}
+	if backupLocation.Spec.Credential != nil {
+		credsFile, err := credentialFileStore.Path(backupLocation.Spec.Credential)
+		if err != nil {
+			return map[string]string{}, errors.WithStack(err)
+		}
+		config[repoconfig.CredentialsFileKey] = credsFile
+	}
 
 	bucket := strings.Trim(config["bucket"], "/")
 	prefix := strings.Trim(config["prefix"], "/")
@@ -477,9 +485,11 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo
 
 		var err error
 		if s3URL == "" {
-			region, err = getS3BucketRegion(bucket)
-			if err != nil {
-				return map[string]string{}, errors.Wrap(err, "error get s3 bucket region")
+			if region == "" {
+				region, err = getS3BucketRegion(bucket)
+				if err != nil {
+					return map[string]string{}, errors.Wrap(err, "error get s3 bucket region")
+				}
 			}
 
 			s3URL = fmt.Sprintf("s3-%s.amazonaws.com", region)

pkg/repository/provider/unified_repo_test.go

@@ -22,7 +22,7 @@ import (
 	"errors"
 	"testing"
 
-	awscredentials "github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/kopia/kopia/repo"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -45,7 +45,7 @@ func TestGetStorageCredentials(t *testing.T) {
 		credFileStore     *credmock.FileStore
 		credStoreError    error
 		credStorePath     string
-		getS3Credentials  func(map[string]string) (*awscredentials.Value, error)
+		getS3Credentials  func(map[string]string) (*aws.Credentials, error)
 		getGCPCredentials func(map[string]string) string
 		expected          map[string]string
 		expectedErr       string
@@ -89,8 +89,8 @@ func TestGetStorageCredentials(t *testing.T) {
 					},
 				},
 			},
-			getS3Credentials: func(config map[string]string) (*awscredentials.Value, error) {
-				return &awscredentials.Value{
+			getS3Credentials: func(config map[string]string) (*aws.Credentials, error) {
+				return &aws.Credentials{
 					AccessKeyID: "from: " + config["credentialsFile"],
 				}, nil
 			},
@@ -115,8 +115,8 @@ func TestGetStorageCredentials(t *testing.T) {
 			},
 			credFileStore: new(credmock.FileStore),
 			credStorePath: "credentials-from-credential-key",
-			getS3Credentials: func(config map[string]string) (*awscredentials.Value, error) {
-				return &awscredentials.Value{
+			getS3Credentials: func(config map[string]string) (*aws.Credentials, error) {
+				return &aws.Credentials{
 					AccessKeyID: "from: " + config["credentialsFile"],
 				}, nil
 			},
@@ -138,7 +138,7 @@ func TestGetStorageCredentials(t *testing.T) {
 					},
 				},
 			},
-			getS3Credentials: func(config map[string]string) (*awscredentials.Value, error) {
+			getS3Credentials: func(config map[string]string) (*aws.Credentials, error) {
 				return nil, errors.New("fake error")
 			},
 			credFileStore: new(credmock.FileStore),
@@ -153,7 +153,7 @@ func TestGetStorageCredentials(t *testing.T) {
 					Config:   map[string]string{},
 				},
 			},
-			getS3Credentials: func(config map[string]string) (*awscredentials.Value, error) {
+			getS3Credentials: func(config map[string]string) (*aws.Credentials, error) {
 				return nil, nil
 			},
 			credFileStore: new(credmock.FileStore),
@@ -437,11 +437,12 @@ func TestGetStorageVariables(t *testing.T) {
 		},
 	}
 
+	credFileStore := new(credmock.FileStore)
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			getS3BucketRegion = tc.getS3BucketRegion
 
-			actual, err := getStorageVariables(&tc.backupLocation, tc.repoBackend, tc.repoName)
+			actual, err := getStorageVariables(&tc.backupLocation, tc.repoBackend, tc.repoName, credFileStore)
 
 			require.Equal(t, tc.expected, actual)
 
@@ -530,7 +531,7 @@ func TestGetStoreOptions(t *testing.T) {
 				BackupRepo:     &velerov1api.BackupRepository{},
 			},
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, errors.New("fake-error-2")
 				},
 			},
@@ -544,7 +545,7 @@ func TestGetStoreOptions(t *testing.T) {
 				BackupRepo:     &velerov1api.BackupRepository{},
 			},
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -604,7 +605,7 @@ func TestPrepareRepo(t *testing.T) {
 			repoService:     new(reposervicenmocks.BackupRepoService),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, errors.New("fake-store-option-error")
 				},
 			},
@@ -615,7 +616,7 @@ func TestPrepareRepo(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -635,7 +636,7 @@ func TestPrepareRepo(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -656,7 +657,7 @@ func TestPrepareRepo(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -733,7 +734,7 @@ func TestForget(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -757,7 +758,7 @@ func TestForget(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -785,7 +786,7 @@ func TestForget(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -877,7 +878,7 @@ func TestInitRepo(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -895,7 +896,7 @@ func TestInitRepo(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -965,7 +966,7 @@ func TestConnectToRepo(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -983,7 +984,7 @@ func TestConnectToRepo(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1057,7 +1058,7 @@ func TestBoostRepoConnect(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1084,7 +1085,7 @@ func TestBoostRepoConnect(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1110,7 +1111,7 @@ func TestBoostRepoConnect(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1197,7 +1198,7 @@ func TestPruneRepo(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1215,7 +1216,7 @@ func TestPruneRepo(t *testing.T) {
 			getter:          new(credmock.SecretStore),
 			credStoreReturn: "fake-password",
 			funcTable: localFuncTable{
-				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
 					return map[string]string{}, nil
 				},
 				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {

pkg/restore/pod_volume_restore_action.go

@@ -27,11 +27,11 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	veleroimage "github.com/vmware-tanzu/velero/internal/velero"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
-	velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/label"
 	"github.com/vmware-tanzu/velero/pkg/plugin/framework/common"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
@@ -47,16 +47,16 @@ const (
 )
 
 type PodVolumeRestoreAction struct {
-	logger                logrus.FieldLogger
-	client                corev1client.ConfigMapInterface
-	podVolumeBackupClient velerov1client.PodVolumeBackupInterface
+	logger   logrus.FieldLogger
+	client   corev1client.ConfigMapInterface
+	crClient ctrlclient.Client
 }
 
-func NewPodVolumeRestoreAction(logger logrus.FieldLogger, client corev1client.ConfigMapInterface, podVolumeBackupClient velerov1client.PodVolumeBackupInterface) *PodVolumeRestoreAction {
+func NewPodVolumeRestoreAction(logger logrus.FieldLogger, client corev1client.ConfigMapInterface, crClient ctrlclient.Client) *PodVolumeRestoreAction {
 	return &PodVolumeRestoreAction{
-		logger:                logger,
-		client:                client,
-		podVolumeBackupClient: podVolumeBackupClient,
+		logger:   logger,
+		client:   client,
+		crClient: crClient,
 	}
 }
 
@@ -86,9 +86,11 @@ func (a *PodVolumeRestoreAction) Execute(input *velero.RestoreItemActionExecuteI
 
 	log := a.logger.WithField("pod", kube.NamespaceAndName(&pod))
 
-	opts := label.NewListOptionsForBackup(input.Restore.Spec.BackupName)
-	podVolumeBackupList, err := a.podVolumeBackupClient.List(context.TODO(), opts)
-	if err != nil {
+	opts := &ctrlclient.ListOptions{
+		LabelSelector: label.NewSelectorForBackup(input.Restore.Spec.BackupName),
+	}
+	podVolumeBackupList := new(velerov1api.PodVolumeBackupList)
+	if err := a.crClient.List(context.TODO(), podVolumeBackupList, opts); err != nil {
 		return nil, errors.WithStack(err)
 	}
 

pkg/restore/pod_volume_restore_action_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package restore
 
 import (
-	"context"
 	"sort"
 	"testing"
 
@@ -25,7 +24,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1api "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/fake"
@@ -34,7 +32,6 @@ import (
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	"github.com/vmware-tanzu/velero/pkg/buildinfo"
-	velerofake "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
 	"github.com/vmware-tanzu/velero/pkg/util/kube"
@@ -131,7 +128,7 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) {
 		name             string
 		pod              *corev1api.Pod
 		podFromBackup    *corev1api.Pod
-		podVolumeBackups []*velerov1api.PodVolumeBackup
+		podVolumeBackups []runtime.Object
 		want             *corev1api.Pod
 	}{
 		{
@@ -179,7 +176,7 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) {
 					builder.WithAnnotations("snapshot.velero.io/not-used", "")).
 				InitContainers(builder.ForContainer("first-container", "").Result()).
 				Result(),
-			podVolumeBackups: []*velerov1api.PodVolumeBackup{
+			podVolumeBackups: []runtime.Object{
 				builder.ForPodVolumeBackup(veleroNs, "pvb-1").
 					PodName("my-pod").
 					PodNamespace("ns-1").
@@ -225,7 +222,7 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) {
 					builder.ForVolume("vol-2").PersistentVolumeClaimSource("pvc-2").Result(),
 				).
 				Result(),
-			podVolumeBackups: []*velerov1api.PodVolumeBackup{
+			podVolumeBackups: []runtime.Object{
 				builder.ForPodVolumeBackup(veleroNs, "pvb-1").
 					PodName("my-pod").
 					PodNamespace("original-ns").
@@ -259,12 +256,7 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			clientset := fake.NewSimpleClientset()
-			clientsetVelero := velerofake.NewSimpleClientset()
-
-			for _, podVolumeBackup := range tc.podVolumeBackups {
-				_, err := clientsetVelero.VeleroV1().PodVolumeBackups(veleroNs).Create(context.TODO(), podVolumeBackup, metav1.CreateOptions{})
-				require.NoError(t, err)
-			}
+			crClient := velerotest.NewFakeControllerRuntimeClient(t, tc.podVolumeBackups...)
 
 			unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pod)
 			require.NoError(t, err)
@@ -294,7 +286,7 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) {
 			a := NewPodVolumeRestoreAction(
 				logrus.StandardLogger(),
 				clientset.CoreV1().ConfigMaps(veleroNs),
-				clientsetVelero.VeleroV1().PodVolumeBackups(veleroNs),
+				crClient,
 			)
 
 			// method under test

pkg/restore/pv_restorer_test.go

@@ -28,8 +28,6 @@ import (
 
 	api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
-	"github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake"
-	informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions"
 	providermocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/volumesnapshotter/v1"
 	vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
@@ -116,11 +114,6 @@ func TestExecutePVAction_NoSnapshotRestores(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			var (
-				client                   = fake.NewSimpleClientset()
-				snapshotLocationInformer = informers.NewSharedInformerFactory(client, 0).Velero().V1().VolumeSnapshotLocations()
-			)
-
 			r := &pvRestorer{
 				logger:     velerotest.NewLogger(),
 				restorePVs: tc.restore.Spec.RestorePVs,
@@ -132,7 +125,7 @@ func TestExecutePVAction_NoSnapshotRestores(t *testing.T) {
 			}
 
 			for _, loc := range tc.locations {
-				require.NoError(t, snapshotLocationInformer.Informer().GetStore().Add(loc))
+				require.NoError(t, r.kbclient.Create(context.TODO(), loc))
 			}
 
 			res, err := r.executePVAction(tc.obj)

pkg/restore/request.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"sort"
 
+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/runtime"
 
@@ -60,6 +61,7 @@ type Request struct {
 	itemOperationsList   *[]*itemoperation.RestoreOperation
 	ResourceModifiers    *resourcemodifiers.ResourceModifiers
 	DisableInformerCache bool
+	CSIVolumeSnapshots   []*snapshotv1api.VolumeSnapshot
 }
 
 type restoredItemStatus struct {

pkg/restore/restore.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
@@ -68,6 +69,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/podvolume"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
+	csiutil "github.com/vmware-tanzu/velero/pkg/util/csi"
 	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
 	"github.com/vmware-tanzu/velero/pkg/util/kube"
 	"github.com/vmware-tanzu/velero/pkg/util/results"
@@ -113,6 +115,7 @@ type kubernetesRestorer struct {
 	podGetter                  cache.Getter
 	credentialFileStore        credentials.FileStore
 	kbClient                   crclient.Client
+	featureVerifier            features.Verifier
 }
 
 // NewKubernetesRestorer creates a new kubernetesRestorer.
@@ -130,6 +133,7 @@ func NewKubernetesRestorer(
 	podGetter cache.Getter,
 	credentialStore credentials.FileStore,
 	kbClient crclient.Client,
+	featureVerifier features.Verifier,
 ) (Restorer, error) {
 	return &kubernetesRestorer{
 		discoveryHelper:            discoveryHelper,
@@ -154,6 +158,7 @@ func NewKubernetesRestorer(
 		podGetter:           podGetter,
 		credentialFileStore: credentialStore,
 		kbClient:            kbClient,
+		featureVerifier:     featureVerifier,
 	}, nil
 }
 
@@ -298,6 +303,7 @@ func (kr *kubernetesRestorer) RestoreWithResolvers(
 		pvsToProvision:                 sets.NewString(),
 		pvRestorer:                     pvRestorer,
 		volumeSnapshots:                req.VolumeSnapshots,
+		csiVolumeSnapshots:             req.CSIVolumeSnapshots,
 		podVolumeBackups:               req.PodVolumeBackups,
 		resourceTerminatingTimeout:     kr.resourceTerminatingTimeout,
 		resourceTimeout:                kr.resourceTimeout,
@@ -310,7 +316,7 @@ func (kr *kubernetesRestorer) RestoreWithResolvers(
 		discoveryHelper:                kr.discoveryHelper,
 		resourcePriorities:             kr.resourcePriorities,
 		resourceRestoreHooks:           resourceRestoreHooks,
-		hooksErrs:                      make(chan error),
+		hooksErrs:                      make(chan hook.HookErrInfo),
 		waitExecHookHandler:            waitExecHookHandler,
 		hooksContext:                   hooksCtx,
 		hooksCancelFunc:                hooksCancelFunc,
@@ -318,6 +324,7 @@ func (kr *kubernetesRestorer) RestoreWithResolvers(
 		itemOperationsList:             req.GetItemOperationsList(),
 		resourceModifiers:              req.ResourceModifiers,
 		disableInformerCache:           req.DisableInformerCache,
+		featureVerifier:                kr.featureVerifier,
 	}
 
 	return restoreCtx.execute()
@@ -347,6 +354,7 @@ type restoreContext struct {
 	pvsToProvision                 sets.String
 	pvRestorer                     PVRestorer
 	volumeSnapshots                []*volume.Snapshot
+	csiVolumeSnapshots             []*snapshotv1api.VolumeSnapshot
 	podVolumeBackups               []*velerov1api.PodVolumeBackup
 	resourceTerminatingTimeout     time.Duration
 	resourceTimeout                time.Duration
@@ -359,7 +367,7 @@ type restoreContext struct {
 	discoveryHelper                discovery.Helper
 	resourcePriorities             Priorities
 	hooksWaitGroup                 sync.WaitGroup
-	hooksErrs                      chan error
+	hooksErrs                      chan hook.HookErrInfo
 	resourceRestoreHooks           []hook.ResourceRestoreHook
 	waitExecHookHandler            hook.WaitExecHookHandler
 	hooksContext                   go_context.Context
@@ -368,6 +376,7 @@ type restoreContext struct {
 	itemOperationsList             *[]*itemoperation.RestoreOperation
 	resourceModifiers              *resourcemodifiers.ResourceModifiers
 	disableInformerCache           bool
+	featureVerifier                features.Verifier
 }
 
 type resourceClientKey struct {
@@ -654,8 +663,8 @@ func (ctx *restoreContext) execute() (results.Result, results.Result) {
 		ctx.hooksWaitGroup.Wait()
 		close(ctx.hooksErrs)
 	}()
-	for err := range ctx.hooksErrs {
-		errs.Velero = append(errs.Velero, err.Error())
+	for errInfo := range ctx.hooksErrs {
+		errs.Add(errInfo.Namespace, errInfo.Err)
 	}
 	ctx.log.Info("Done waiting for all post-restore exec hooks to complete")
 
@@ -1287,15 +1296,57 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 			}
 
 		case hasPodVolumeBackup(obj, ctx):
-			ctx.log.Infof("Dynamically re-provisioning persistent volume because it has a pod volume backup to be restored.")
+			ctx.log.WithFields(logrus.Fields{
+				"namespace":     obj.GetNamespace(),
+				"name":          obj.GetName(),
+				"groupResource": groupResource.String(),
+			}).Infof("Dynamically re-provisioning persistent volume because it has a pod volume backup to be restored.")
 			ctx.pvsToProvision.Insert(name)
 
 			// Return early because we don't want to restore the PV itself, we
 			// want to dynamically re-provision it.
 			return warnings, errs, itemExists
 
+		case hasCSIVolumeSnapshot(ctx, obj):
+			ctx.log.WithFields(logrus.Fields{
+				"namespace":     obj.GetNamespace(),
+				"name":          obj.GetName(),
+				"groupResource": groupResource.String(),
+			}).Infof("Dynamically re-provisioning persistent volume because it has a related CSI VolumeSnapshot.")
+			ctx.pvsToProvision.Insert(name)
+
+			if ready, err := ctx.featureVerifier.Verify(velerov1api.CSIFeatureFlag); !ready {
+				ctx.log.Errorf("Failed to verify CSI modules, ready %v, err %v", ready, err)
+				errs.Add(namespace, fmt.Errorf("CSI modules are not ready for restore. Check CSI feature is enabled and CSI plugin is installed"))
+			}
+
+			// Return early because we don't want to restore the PV itself, we
+			// want to dynamically re-provision it.
+			return warnings, errs, itemExists
+
+		case hasSnapshotDataUpload(ctx, obj):
+			ctx.log.WithFields(logrus.Fields{
+				"namespace":     obj.GetNamespace(),
+				"name":          obj.GetName(),
+				"groupResource": groupResource.String(),
+			}).Infof("Dynamically re-provisioning persistent volume because it has a related snapshot DataUpload.")
+			ctx.pvsToProvision.Insert(name)
+
+			if ready, err := ctx.featureVerifier.Verify(velerov1api.CSIFeatureFlag); !ready {
+				ctx.log.Errorf("Failed to verify CSI modules, ready %v, err %v", ready, err)
+				errs.Add(namespace, fmt.Errorf("CSI modules are not ready for restore. Check CSI feature is enabled and CSI plugin is installed"))
+			}
+
+			// Return early because we don't want to restore the PV itself, we
+			// want to dynamically re-provision it.
+			return warnings, errs, itemExists
+
 		case hasDeleteReclaimPolicy(obj.Object):
-			ctx.log.Infof("Dynamically re-provisioning persistent volume because it doesn't have a snapshot and its reclaim policy is Delete.")
+			ctx.log.WithFields(logrus.Fields{
+				"namespace":     obj.GetNamespace(),
+				"name":          obj.GetName(),
+				"groupResource": groupResource.String(),
+			}).Infof("Dynamically re-provisioning persistent volume because it doesn't have a snapshot and its reclaim policy is Delete.")
 			ctx.pvsToProvision.Insert(name)
 
 			// Return early because we don't want to restore the PV itself, we
@@ -1303,7 +1354,11 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 			return warnings, errs, itemExists
 
 		default:
-			ctx.log.Infof("Restoring persistent volume as-is because it doesn't have a snapshot and its reclaim policy is not Delete.")
+			ctx.log.WithFields(logrus.Fields{
+				"namespace":     obj.GetNamespace(),
+				"name":          obj.GetName(),
+				"groupResource": groupResource.String(),
+			}).Infof("Restoring persistent volume as-is because it doesn't have a snapshot and its reclaim policy is not Delete.")
 
 			// Check to see if the claimRef.namespace field needs to be remapped, and do so if necessary.
 			_, err = remapClaimRefNS(ctx, obj)
@@ -1337,6 +1392,13 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 			continue
 		}
 
+		// If the EnableCSI feature is not enabled, but the executing action is from CSI plugin, skip the action.
+		if csiutil.ShouldSkipAction(action.Name()) {
+			ctx.log.Infof("Skip action %s for resource %s:%s/%s, because the CSI feature is not enabled. Feature setting is %s.",
+				action.Name(), groupResource.String(), obj.GetNamespace(), obj.GetName(), features.Serialize())
+			continue
+		}
+
 		ctx.log.Infof("Executing item action for %v", &groupResource)
 		executeOutput, err := action.RestoreItemAction.Execute(&velero.RestoreItemActionExecuteInput{
 			Item:           obj,
@@ -1466,7 +1528,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 	}
 
 	if ctx.resourceModifiers != nil {
-		if errList := ctx.resourceModifiers.ApplyResourceModifierRules(obj, groupResource.String(), ctx.log); errList != nil {
+		if errList := ctx.resourceModifiers.ApplyResourceModifierRules(obj, groupResource.String(), ctx.kbClient.Scheme(), ctx.log); errList != nil {
 			for _, err := range errList {
 				errs.Add(namespace, err)
 			}
@@ -1524,18 +1586,17 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 	}
 
 	if restoreErr != nil {
-		// check for the existence of the object in cluster, if no error then it implies that object exists
-		// and if err then we want to judge whether there is an existing error in the previous creation.
-		// if so, we will return the 'get' error.
-		// otherwise, we will return the original creation error.
+		// check for the existence of the object that failed creation due to alreadyExist in cluster, if no error then it implies that object exists.
+		// and if err then itemExists remains false as we were not able to confirm the existence of the object via Get call or creation call.
+		// We return the get error as a warning to notify the user that the object could exist in cluster and we were not able to confirm it.
 		if !ctx.disableInformerCache {
 			fromCluster, err = ctx.getResource(groupResource, obj, namespace, name)
 		} else {
 			fromCluster, err = resourceClient.Get(name, metav1.GetOptions{})
 		}
 		if err != nil && isAlreadyExistsError {
-			ctx.log.Errorf("Error retrieving in-cluster version of %s: %v", kube.NamespaceAndName(obj), err)
-			errs.Add(namespace, err)
+			ctx.log.Warnf("Unable to retrieve in-cluster version of %s: %v, object won't be restored by velero or have restore labels, and existing resource policy is not applied", kube.NamespaceAndName(obj), err)
+			warnings.Add(namespace, err)
 			return warnings, errs, itemExists
 		}
 	}
@@ -1891,10 +1952,11 @@ func (ctx *restoreContext) waitExec(createdObj *unstructured.Unstructured) {
 		// on the ctx.podVolumeErrs channel.
 		defer ctx.hooksWaitGroup.Done()
 
+		podNs := createdObj.GetNamespace()
 		pod := new(v1.Pod)
 		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(createdObj.UnstructuredContent(), &pod); err != nil {
 			ctx.log.WithError(err).Error("error converting unstructured pod")
-			ctx.hooksErrs <- err
+			ctx.hooksErrs <- hook.HookErrInfo{Namespace: podNs, Err: err}
 			return
 		}
 		execHooksByContainer, err := hook.GroupRestoreExecHooks(
@@ -1904,7 +1966,7 @@ func (ctx *restoreContext) waitExec(createdObj *unstructured.Unstructured) {
 		)
 		if err != nil {
 			ctx.log.WithError(err).Errorf("error getting exec hooks for pod %s/%s", pod.Namespace, pod.Name)
-			ctx.hooksErrs <- err
+			ctx.hooksErrs <- hook.HookErrInfo{Namespace: podNs, Err: err}
 			return
 		}
 
@@ -1914,7 +1976,7 @@ func (ctx *restoreContext) waitExec(createdObj *unstructured.Unstructured) {
 
 			for _, err := range errs {
 				// Errors are already logged in the HandleHooks method.
-				ctx.hooksErrs <- err
+				ctx.hooksErrs <- hook.HookErrInfo{Namespace: podNs, Err: err}
 			}
 		}
 	}()
@@ -1930,6 +1992,55 @@ func hasSnapshot(pvName string, snapshots []*volume.Snapshot) bool {
 	return false
 }
 
+func hasCSIVolumeSnapshot(ctx *restoreContext, unstructuredPV *unstructured.Unstructured) bool {
+	pv := new(v1.PersistentVolume)
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.Object, pv); err != nil {
+		ctx.log.WithError(err).Warnf("Unable to convert PV from unstructured to structured")
+		return false
+	}
+
+	for _, vs := range ctx.csiVolumeSnapshots {
+		if pv.Spec.ClaimRef.Name == *vs.Spec.Source.PersistentVolumeClaimName &&
+			pv.Spec.ClaimRef.Namespace == vs.Namespace {
+			return true
+		}
+	}
+	return false
+}
+
+func hasSnapshotDataUpload(ctx *restoreContext, unstructuredPV *unstructured.Unstructured) bool {
+	pv := new(v1.PersistentVolume)
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.Object, pv); err != nil {
+		ctx.log.WithError(err).Warnf("Unable to convert PV from unstructured to structured")
+		return false
+	}
+
+	if pv.Spec.ClaimRef == nil {
+		return false
+	}
+
+	dataUploadResultList := new(v1.ConfigMapList)
+	err := ctx.kbClient.List(go_context.TODO(), dataUploadResultList, &crclient.ListOptions{
+		LabelSelector: labels.SelectorFromSet(map[string]string{
+			velerov1api.RestoreUIDLabel:       label.GetValidName(string(ctx.restore.GetUID())),
+			velerov1api.PVCNamespaceNameLabel: label.GetValidName(pv.Spec.ClaimRef.Namespace + "." + pv.Spec.ClaimRef.Name),
+			velerov1api.ResourceUsageLabel:    label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)),
+		}),
+	})
+	if err != nil {
+		ctx.log.WithError(err).Warnf("Fail to list DataUpload result CM.")
+		return false
+	}
+
+	if len(dataUploadResultList.Items) != 1 {
+		ctx.log.WithError(fmt.Errorf("dataupload result number is not expected")).
+			Warnf("Got %d DataUpload result. Expect one.", len(dataUploadResultList.Items))
+		return false
+	}
+
+	return true
+}
+
 func hasPodVolumeBackup(unstructuredPV *unstructured.Unstructured, ctx *restoreContext) bool {
 	if len(ctx.podVolumeBackups) == 0 {
 		return false

pkg/restore/restore_test.go

@@ -25,9 +25,11 @@ import (
 	"testing"
 	"time"
 
+	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 	corev1api "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -45,6 +47,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	"github.com/vmware-tanzu/velero/pkg/client"
 	"github.com/vmware-tanzu/velero/pkg/discovery"
+	verifiermocks "github.com/vmware-tanzu/velero/pkg/features/mocks"
 	"github.com/vmware-tanzu/velero/pkg/itemoperation"
 	"github.com/vmware-tanzu/velero/pkg/kuberesource"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
@@ -2256,6 +2259,7 @@ func (*volumeSnapshotter) DeleteSnapshot(snapshotID string) error {
 // Verification is done by looking at the contents of the API and the metadata/spec/status of
 // the items in the API.
 func TestRestorePersistentVolumes(t *testing.T) {
+	testPVCName := "testPVC"
 	tests := []struct {
 		name                    string
 		restore                 *velerov1api.Restore
@@ -2265,9 +2269,12 @@ func TestRestorePersistentVolumes(t *testing.T) {
 		volumeSnapshots         []*volume.Snapshot
 		volumeSnapshotLocations []*velerov1api.VolumeSnapshotLocation
 		volumeSnapshotterGetter volumeSnapshotterGetter
+		csiVolumeSnapshots      []*snapshotv1api.VolumeSnapshot
+		dataUploadResult        *corev1api.ConfigMap
 		want                    []*test.APIResource
 		wantError               bool
 		wantWarning             bool
+		csiFeatureVerifierErr   string
 	}{
 		{
 			name:    "when a PV with a reclaim policy of delete has no snapshot and does not exist in-cluster, it does not get restored, and its PVC gets reset for dynamic provisioning",
@@ -2923,6 +2930,152 @@ func TestRestorePersistentVolumes(t *testing.T) {
 				),
 			},
 		},
+		{
+			name:    "when a PV with a reclaim policy of retain has a CSI VolumeSnapshot and does not exist in-cluster, the PV is not restored",
+			restore: defaultRestore().Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("persistentvolumes",
+					builder.ForPersistentVolume("pv-1").
+						ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).
+						ClaimRef("velero", testPVCName).
+						Result(),
+				).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.PVs(),
+				test.PVCs(),
+			},
+			csiVolumeSnapshots: []*snapshotv1api.VolumeSnapshot{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "velero",
+						Name:      "test",
+					},
+					Spec: snapshotv1api.VolumeSnapshotSpec{
+						Source: snapshotv1api.VolumeSnapshotSource{
+							PersistentVolumeClaimName: &testPVCName,
+						},
+					},
+				},
+			},
+			volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{
+				builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(),
+			},
+			volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{
+				"provider-1": &volumeSnapshotter{
+					snapshotVolumes: map[string]string{"snapshot-1": "new-volume"},
+				},
+			},
+			want: []*test.APIResource{},
+		},
+		{
+			name:    "when a PV has a CSI VolumeSnapshot, but CSI modules are not ready, the PV is not restored",
+			restore: defaultRestore().Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("persistentvolumes",
+					builder.ForPersistentVolume("pv-1").
+						ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).
+						ClaimRef("velero", testPVCName).
+						Result(),
+				).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.PVs(),
+				test.PVCs(),
+			},
+			csiVolumeSnapshots: []*snapshotv1api.VolumeSnapshot{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "velero",
+						Name:      "test",
+					},
+					Spec: snapshotv1api.VolumeSnapshotSpec{
+						Source: snapshotv1api.VolumeSnapshotSource{
+							PersistentVolumeClaimName: &testPVCName,
+						},
+					},
+				},
+			},
+			volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{
+				builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(),
+			},
+			volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{
+				"provider-1": &volumeSnapshotter{
+					snapshotVolumes: map[string]string{"snapshot-1": "new-volume"},
+				},
+			},
+			want:                  []*test.APIResource{},
+			csiFeatureVerifierErr: "fake-feature-check-error",
+			wantError:             true,
+		},
+		{
+			name:    "when a PV with a reclaim policy of retain has a DataUpload result CM and does not exist in-cluster, the PV is not restored",
+			restore: defaultRestore().ObjectMeta(builder.WithUID("fakeUID")).Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("persistentvolumes",
+					builder.ForPersistentVolume("pv-1").
+						ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).
+						ClaimRef("velero", testPVCName).
+						Result(),
+				).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.PVs(),
+				test.PVCs(),
+				test.ConfigMaps(),
+			},
+			volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{
+				builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(),
+			},
+			volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{
+				"provider-1": &volumeSnapshotter{
+					snapshotVolumes: map[string]string{"snapshot-1": "new-volume"},
+				},
+			},
+			dataUploadResult: builder.ForConfigMap("velero", "test").ObjectMeta(builder.WithLabelsMap(map[string]string{
+				velerov1api.RestoreUIDLabel:       "fakeUID",
+				velerov1api.PVCNamespaceNameLabel: "velero.testPVC",
+				velerov1api.ResourceUsageLabel:    string(velerov1api.VeleroResourceUsageDataUploadResult),
+			})).Result(),
+			want: []*test.APIResource{},
+		},
+		{
+			name:    "when a PV has a DataUpload result CM, but CSI modules are not ready, the PV is not restored",
+			restore: defaultRestore().ObjectMeta(builder.WithUID("fakeUID")).Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("persistentvolumes",
+					builder.ForPersistentVolume("pv-1").
+						ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).
+						ClaimRef("velero", testPVCName).
+						Result(),
+				).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.PVs(),
+				test.PVCs(),
+				test.ConfigMaps(),
+			},
+			volumeSnapshotLocations: []*velerov1api.VolumeSnapshotLocation{
+				builder.ForVolumeSnapshotLocation(velerov1api.DefaultNamespace, "default").Provider("provider-1").Result(),
+			},
+			volumeSnapshotterGetter: map[string]vsv1.VolumeSnapshotter{
+				"provider-1": &volumeSnapshotter{
+					snapshotVolumes: map[string]string{"snapshot-1": "new-volume"},
+				},
+			},
+			dataUploadResult: builder.ForConfigMap("velero", "test").ObjectMeta(builder.WithLabelsMap(map[string]string{
+				velerov1api.RestoreUIDLabel:       "fakeUID",
+				velerov1api.PVCNamespaceNameLabel: "velero.testPVC",
+				velerov1api.ResourceUsageLabel:    string(velerov1api.VeleroResourceUsageDataUploadResult),
+			})).Result(),
+			want:                  []*test.APIResource{},
+			csiFeatureVerifierErr: "fake-feature-check-error",
+			wantError:             true,
+		},
 	}
 
 	for _, tc := range tests {
@@ -2934,11 +3087,23 @@ func TestRestorePersistentVolumes(t *testing.T) {
 				return renamed, nil
 			}
 
+			verifierMock := new(verifiermocks.Verifier)
+			if tc.csiFeatureVerifierErr != "" {
+				verifierMock.On("Verify", mock.Anything, mock.Anything).Return(false, errors.New(tc.csiFeatureVerifierErr))
+			} else {
+				verifierMock.On("Verify", mock.Anything, mock.Anything).Return(true, nil)
+			}
+			h.restorer.featureVerifier = verifierMock
+
 			// set up the VolumeSnapshotLocation client and add test data to it
 			for _, vsl := range tc.volumeSnapshotLocations {
 				require.NoError(t, h.restorer.kbClient.Create(context.Background(), vsl))
 			}
 
+			if tc.dataUploadResult != nil {
+				require.NoError(t, h.restorer.kbClient.Create(context.TODO(), tc.dataUploadResult))
+			}
+
 			for _, r := range tc.apiResources {
 				h.AddItems(t, r)
 			}
@@ -2955,11 +3120,12 @@ func TestRestorePersistentVolumes(t *testing.T) {
 			}
 
 			data := &Request{
-				Log:             h.log,
-				Restore:         tc.restore,
-				Backup:          tc.backup,
-				VolumeSnapshots: tc.volumeSnapshots,
-				BackupReader:    tc.tarball,
+				Log:                h.log,
+				Restore:            tc.restore,
+				Backup:             tc.backup,
+				VolumeSnapshots:    tc.volumeSnapshots,
+				BackupReader:       tc.tarball,
+				CSIVolumeSnapshots: tc.csiVolumeSnapshots,
 			}
 			warnings, errs := h.restorer.Restore(
 				data,
@@ -3652,3 +3818,175 @@ func TestIsAlreadyExistsError(t *testing.T) {
 		})
 	}
 }
+
+func TestHasCSIVolumeSnapshot(t *testing.T) {
+	tests := []struct {
+		name           string
+		vs             *snapshotv1api.VolumeSnapshot
+		obj            *unstructured.Unstructured
+		expectedResult bool
+	}{
+		{
+			name: "Invalid PV, expect false.",
+			obj: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"kind": 1,
+				},
+			},
+			expectedResult: false,
+		},
+		{
+			name: "Cannot find VS, expect false",
+			obj: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"kind":       "PersistentVolume",
+					"apiVersion": "v1",
+					"metadata": map[string]interface{}{
+						"namespace": "default",
+						"name":      "test",
+					},
+				},
+			},
+			expectedResult: false,
+		},
+		{
+			name: "Find VS, expect true.",
+			obj: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"kind":       "PersistentVolume",
+					"apiVersion": "v1",
+					"metadata": map[string]interface{}{
+						"namespace": "velero",
+						"name":      "test",
+					},
+					"spec": map[string]interface{}{
+						"claimRef": map[string]interface{}{
+							"namespace": "velero",
+							"name":      "test",
+						},
+					},
+				},
+			},
+			vs:             builder.ForVolumeSnapshot("velero", "test").SourcePVC("test").Result(),
+			expectedResult: true,
+		},
+	}
+
+	for _, tc := range tests {
+		h := newHarness(t)
+
+		ctx := &restoreContext{
+			log: h.log,
+		}
+
+		if tc.vs != nil {
+			ctx.csiVolumeSnapshots = []*snapshotv1api.VolumeSnapshot{tc.vs}
+		}
+
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.expectedResult, hasCSIVolumeSnapshot(ctx, tc.obj))
+		})
+	}
+}
+
+func TestHasSnapshotDataUpload(t *testing.T) {
+	tests := []struct {
+		name           string
+		duResult       *corev1api.ConfigMap
+		obj            *unstructured.Unstructured
+		expectedResult bool
+		restore        *velerov1api.Restore
+	}{
+		{
+			name: "Invalid PV, expect false.",
+			obj: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"kind": 1,
+				},
+			},
+			expectedResult: false,
+		},
+		{
+			name: "PV without ClaimRef, expect false",
+			obj: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"kind":       "PersistentVolume",
+					"apiVersion": "v1",
+					"metadata": map[string]interface{}{
+						"namespace": "default",
+						"name":      "test",
+					},
+				},
+			},
+			duResult:       builder.ForConfigMap("velero", "test").Result(),
+			restore:        builder.ForRestore("velero", "test").ObjectMeta(builder.WithUID("fakeUID")).Result(),
+			expectedResult: false,
+		},
+		{
+			name: "Cannot find DataUploadResult CM, expect false",
+			obj: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"kind":       "PersistentVolume",
+					"apiVersion": "v1",
+					"metadata": map[string]interface{}{
+						"namespace": "default",
+						"name":      "test",
+					},
+					"spec": map[string]interface{}{
+						"claimRef": map[string]interface{}{
+							"namespace": "velero",
+							"name":      "testPVC",
+						},
+					},
+				},
+			},
+			duResult:       builder.ForConfigMap("velero", "test").Result(),
+			restore:        builder.ForRestore("velero", "test").ObjectMeta(builder.WithUID("fakeUID")).Result(),
+			expectedResult: false,
+		},
+		{
+			name: "Find DataUploadResult CM, expect true",
+			obj: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"kind":       "PersistentVolume",
+					"apiVersion": "v1",
+					"metadata": map[string]interface{}{
+						"namespace": "default",
+						"name":      "test",
+					},
+					"spec": map[string]interface{}{
+						"claimRef": map[string]interface{}{
+							"namespace": "velero",
+							"name":      "testPVC",
+						},
+					},
+				},
+			},
+			duResult: builder.ForConfigMap("velero", "test").ObjectMeta(builder.WithLabelsMap(map[string]string{
+				velerov1api.RestoreUIDLabel:       "fakeUID",
+				velerov1api.PVCNamespaceNameLabel: "velero/testPVC",
+				velerov1api.ResourceUsageLabel:    string(velerov1api.VeleroResourceUsageDataUploadResult),
+			})).Result(),
+			restore:        builder.ForRestore("velero", "test").ObjectMeta(builder.WithUID("fakeUID")).Result(),
+			expectedResult: false,
+		},
+	}
+
+	for _, tc := range tests {
+		h := newHarness(t)
+
+		ctx := &restoreContext{
+			log:      h.log,
+			kbClient: h.restorer.kbClient,
+			restore:  tc.restore,
+		}
+
+		if tc.duResult != nil {
+			require.NoError(t, ctx.kbClient.Create(context.TODO(), tc.duResult))
+		}
+
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.expectedResult, hasSnapshotDataUpload(ctx, tc.obj))
+		})
+	}
+}

pkg/restore/service_action.go

@@ -66,6 +66,9 @@ func (a *ServiceAction) Execute(input *velero.RestoreItemActionExecuteInput) (*v
 		if err := deleteNodePorts(service); err != nil {
 			return nil, err
 		}
+		if err := deleteHealthCheckNodePort(service); err != nil {
+			return nil, err
+		}
 	}
 
 	res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(service)
@@ -76,6 +79,72 @@ func (a *ServiceAction) Execute(input *velero.RestoreItemActionExecuteInput) (*v
 	return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: res}), nil
 }
 
+func deleteHealthCheckNodePort(service *corev1api.Service) error {
+	// Check service type and external traffic policy setting,
+	// if the setting is not applicable for HealthCheckNodePort, return early.
+	if service.Spec.ExternalTrafficPolicy != corev1api.ServiceExternalTrafficPolicyTypeLocal ||
+		service.Spec.Type != corev1api.ServiceTypeLoadBalancer {
+		return nil
+	}
+
+	// HealthCheckNodePort is already 0, return.
+	if service.Spec.HealthCheckNodePort == 0 {
+		return nil
+	}
+
+	// Search HealthCheckNodePort from server's last-applied-configuration
+	// annotation(HealthCheckNodePort is specified by `kubectl apply` command)
+	lastAppliedConfig, ok := service.Annotations[annotationLastAppliedConfig]
+	if ok {
+		appliedServiceUnstructured := new(map[string]interface{})
+		if err := json.Unmarshal([]byte(lastAppliedConfig), appliedServiceUnstructured); err != nil {
+			return errors.WithStack(err)
+		}
+
+		healthCheckNodePort, exist, err := unstructured.NestedFloat64(*appliedServiceUnstructured, "spec", "healthCheckNodePort")
+		if err != nil {
+			return errors.WithStack(err)
+		}
+
+		// Found healthCheckNodePort in lastAppliedConfig annotation,
+		// and the value is not 0. No need to delete, return.
+		if exist && healthCheckNodePort != 0 {
+			return nil
+		}
+	}
+
+	// Search HealthCheckNodePort from ManagedFields(HealthCheckNodePort
+	// is specified by `kubectl apply --server-side` command).
+	for _, entry := range service.GetManagedFields() {
+		if entry.FieldsV1 == nil {
+			continue
+		}
+		fields := new(map[string]interface{})
+		if err := json.Unmarshal(entry.FieldsV1.Raw, fields); err != nil {
+			return errors.WithStack(err)
+		}
+
+		_, exist, err := unstructured.NestedMap(*fields, "f:spec", "f:healthCheckNodePort")
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		if !exist {
+			continue
+		}
+		// Because the format in ManagedFields is `f:healthCheckNodePort: {}`,
+		// cannot get the value, check whether exists is enough.
+		// Found healthCheckNodePort in ManagedFields.
+		// No need to delete. Return.
+		return nil
+	}
+
+	// Cannot find HealthCheckNodePort from Annotation and
+	// ManagedFields, which means it's auto-generated. Delete it.
+	service.Spec.HealthCheckNodePort = 0
+
+	return nil
+}
+
 func deleteNodePorts(service *corev1api.Service) error {
 	if service.Spec.Type == corev1api.ServiceTypeExternalName {
 		return nil

pkg/restore/service_action_test.go

@@ -36,7 +36,8 @@ import (
 func svcJSON(ports ...corev1api.ServicePort) string {
 	svc := corev1api.Service{
 		Spec: corev1api.ServiceSpec{
-			Ports: ports,
+			HealthCheckNodePort: 8080,
+			Ports:               ports,
 		},
 	}
 
@@ -486,6 +487,164 @@ func TestServiceActionExecute(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "If PreserveNodePorts is True in restore spec then HealthCheckNodePort always preserved.",
+			obj: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "svc-1",
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   8080,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+					Ports: []corev1api.ServicePort{
+						{
+							Name:     "http",
+							Port:     80,
+							NodePort: 8080,
+						},
+						{
+							Name:     "hepsiburada",
+							NodePort: 9025,
+						},
+					},
+				},
+			},
+			restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(true).Result(),
+			expectedRes: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "svc-1",
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   8080,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+					Ports: []corev1api.ServicePort{
+						{
+							Name:     "http",
+							Port:     80,
+							NodePort: 8080,
+						},
+						{
+							Name:     "hepsiburada",
+							NodePort: 9025,
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "If PreserveNodePorts is False in restore spec then HealthCheckNodePort should be cleaned.",
+			obj: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "svc-1",
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   8080,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+				},
+			},
+			restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(false).Result(),
+			expectedRes: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "svc-1",
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   0,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+				},
+			},
+		},
+		{
+			name: "If PreserveNodePorts is false in restore spec, but service is not expected, then HealthCheckNodePort should be kept.",
+			obj: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "svc-1",
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   8080,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeCluster,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+				},
+			},
+			restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(false).Result(),
+			expectedRes: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "svc-1",
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   8080,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeCluster,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+				},
+			},
+		},
+		{
+			name: "If PreserveNodePorts is false in restore spec, but HealthCheckNodePort can be found in Annotation, then it should be kept.",
+			obj: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        "svc-1",
+					Annotations: map[string]string{annotationLastAppliedConfig: svcJSON()},
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   8080,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+				},
+			},
+			restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(false).Result(),
+			expectedRes: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        "svc-1",
+					Annotations: map[string]string{annotationLastAppliedConfig: svcJSON()},
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   8080,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+				},
+			},
+		},
+		{
+			name: "If PreserveNodePorts is false in restore spec, but HealthCheckNodePort can be found in ManagedFields, then it should be kept.",
+			obj: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "svc-1",
+					ManagedFields: []metav1.ManagedFieldsEntry{
+						{
+							FieldsV1: &metav1.FieldsV1{
+								Raw: []byte(`{"f:spec":{"f:healthCheckNodePort":{}}}`),
+							},
+						},
+					},
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   8080,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+				},
+			},
+			restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(false).Result(),
+			expectedRes: corev1api.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "svc-1",
+					ManagedFields: []metav1.ManagedFieldsEntry{
+						{
+							FieldsV1: &metav1.FieldsV1{
+								Raw: []byte(`{"f:spec":{"f:healthCheckNodePort":{}}}`),
+							},
+						},
+					},
+				},
+				Spec: corev1api.ServiceSpec{
+					HealthCheckNodePort:   8080,
+					ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal,
+					Type:                  corev1api.ServiceTypeLoadBalancer,
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {

pkg/test/api_server.go

@@ -24,14 +24,11 @@ import (
 	discoveryfake "k8s.io/client-go/discovery/fake"
 	dynamicfake "k8s.io/client-go/dynamic/fake"
 	kubefake "k8s.io/client-go/kubernetes/fake"
-
-	"github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake"
 )
 
 // APIServer contains in-memory fakes for all of the relevant
 // Kubernetes API server clients.
 type APIServer struct {
-	VeleroClient    *fake.Clientset
 	KubeClient      *kubefake.Clientset
 	DynamicClient   *dynamicfake.FakeDynamicClient
 	DiscoveryClient *DiscoveryClient
@@ -43,7 +40,6 @@ func NewAPIServer(t *testing.T) *APIServer {
 	t.Helper()
 
 	var (
-		veleroClient  = fake.NewSimpleClientset()
 		kubeClient    = kubefake.NewSimpleClientset()
 		dynamicClient = dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(),
 			map[schema.GroupVersionResource]string{
@@ -65,7 +61,6 @@ func NewAPIServer(t *testing.T) *APIServer {
 	)
 
 	return &APIServer{
-		VeleroClient:    veleroClient,
 		KubeClient:      kubeClient,
 		DynamicClient:   dynamicClient,
 		DiscoveryClient: discoveryClient,

pkg/test/fake_controller_runtime_client.go

@@ -21,6 +21,7 @@ import (
 
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	"github.com/stretchr/testify/require"
+	appsv1api "k8s.io/api/apps/v1"
 	corev1api "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -38,6 +39,8 @@ func NewFakeControllerRuntimeClientBuilder(t *testing.T) *k8sfake.ClientBuilder
 	require.NoError(t, err)
 	err = corev1api.AddToScheme(scheme)
 	require.NoError(t, err)
+	err = appsv1api.AddToScheme(scheme)
+	require.NoError(t, err)
 	err = snapshotv1api.AddToScheme(scheme)
 	require.NoError(t, err)
 	return k8sfake.NewClientBuilder().WithScheme(scheme)
@@ -51,7 +54,13 @@ func NewFakeControllerRuntimeClient(t *testing.T, initObjs ...runtime.Object) cl
 	require.NoError(t, err)
 	err = corev1api.AddToScheme(scheme)
 	require.NoError(t, err)
+	err = appsv1api.AddToScheme(scheme)
+	require.NoError(t, err)
 	err = snapshotv1api.AddToScheme(scheme)
 	require.NoError(t, err)
 	return k8sfake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(initObjs...).Build()
 }
+
+func NewFakeControllerRuntimeWatchClient(t *testing.T, initObjs ...runtime.Object) client.WithWatch {
+	return NewFakeControllerRuntimeClientBuilder(t).WithRuntimeObjects(initObjs...).Build()
+}

pkg/test/mocks.go

@@ -0,0 +1,20 @@
+package test
+
+import (
+	snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
+	snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+// VolumeSnapshotLister helps list VolumeSnapshots.
+// All objects returned here must be treated as read-only.
+//
+//go:generate mockery --name VolumeSnapshotLister
+type VolumeSnapshotLister interface {
+	// List lists all VolumeSnapshots in the indexer.
+	// Objects returned here must be treated as read-only.
+	List(selector labels.Selector) (ret []*snapshotv1.VolumeSnapshot, err error)
+	// VolumeSnapshots returns an object that can list and get VolumeSnapshots.
+	VolumeSnapshots(namespace string) snapshotv1listers.VolumeSnapshotNamespaceLister
+	snapshotv1listers.VolumeSnapshotListerExpansion
+}

pkg/test/mocks/VolumeSnapshotLister.go

@@ -0,0 +1,73 @@
+// Code generated by mockery v2.35.4. DO NOT EDIT.
+
+package mocks
+
+import (
+	mock "github.com/stretchr/testify/mock"
+	labels "k8s.io/apimachinery/pkg/labels"
+
+	v1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
+
+	volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
+)
+
+// VolumeSnapshotLister is an autogenerated mock type for the VolumeSnapshotLister type
+type VolumeSnapshotLister struct {
+	mock.Mock
+}
+
+// List provides a mock function with given fields: selector
+func (_m *VolumeSnapshotLister) List(selector labels.Selector) ([]*v1.VolumeSnapshot, error) {
+	ret := _m.Called(selector)
+
+	var r0 []*v1.VolumeSnapshot
+	var r1 error
+	if rf, ok := ret.Get(0).(func(labels.Selector) ([]*v1.VolumeSnapshot, error)); ok {
+		return rf(selector)
+	}
+	if rf, ok := ret.Get(0).(func(labels.Selector) []*v1.VolumeSnapshot); ok {
+		r0 = rf(selector)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*v1.VolumeSnapshot)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(labels.Selector) error); ok {
+		r1 = rf(selector)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// VolumeSnapshots provides a mock function with given fields: namespace
+func (_m *VolumeSnapshotLister) VolumeSnapshots(namespace string) volumesnapshotv1.VolumeSnapshotNamespaceLister {
+	ret := _m.Called(namespace)
+
+	var r0 volumesnapshotv1.VolumeSnapshotNamespaceLister
+	if rf, ok := ret.Get(0).(func(string) volumesnapshotv1.VolumeSnapshotNamespaceLister); ok {
+		r0 = rf(namespace)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(volumesnapshotv1.VolumeSnapshotNamespaceLister)
+		}
+	}
+
+	return r0
+}
+
+// NewVolumeSnapshotLister creates a new instance of VolumeSnapshotLister. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+// The first argument is typically a *testing.T value.
+func NewVolumeSnapshotLister(t interface {
+	mock.TestingT
+	Cleanup(func())
+}) *VolumeSnapshotLister {
+	mock := &VolumeSnapshotLister{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
+}

pkg/test/resources.go

@@ -142,6 +142,17 @@ func ServiceAccounts(items ...metav1.Object) *APIResource {
 	}
 }
 
+func ConfigMaps(items ...metav1.Object) *APIResource {
+	return &APIResource{
+		Group:      "",
+		Version:    "v1",
+		Name:       "configmaps",
+		ShortName:  "cm",
+		Namespaced: true,
+		Items:      items,
+	}
+}
+
 func CRDs(items ...metav1.Object) *APIResource {
 	return &APIResource{
 		Group:      "apiextensions.k8s.io",

pkg/test/test_logger.go

@@ -40,7 +40,7 @@ type singleLogRecorder struct {
 }
 
 func (s *singleLogRecorder) Write(p []byte) (n int, err error) {
-	*s.buffer = string(p[:])
+	*s.buffer = *s.buffer + string(p[:])
 	return len(p), nil
 }
 

pkg/uploader/kopia/block_backup.go

@@ -1,3 +1,6 @@
+//go:build !windows
+// +build !windows
+
 /*
 Copyright The Velero Contributors.
 

pkg/uploader/kopia/block_backup_windows.go

@@ -0,0 +1,30 @@
+//go:build windows
+// +build windows
+
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kopia
+
+import (
+	"fmt"
+
+	"github.com/kopia/kopia/fs"
+)
+
+func getLocalBlockEntry(sourcePath string) (fs.Entry, error) {
+	return nil, fmt.Errorf("block mode is not supported for Windows")
+}

pkg/uploader/kopia/block_restore.go

@@ -1,3 +1,6 @@
+//go:build !windows
+// +build !windows
+
 /*
 Copyright The Velero Contributors.
 

pkg/uploader/kopia/block_restore_windows.go

@@ -0,0 +1,42 @@
+//go:build windows
+// +build windows
+
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kopia
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/snapshot/restore"
+)
+
+type BlockOutput struct {
+	*restore.FilesystemOutput
+
+	targetFileName string
+}
+
+func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remoteFile fs.File) error {
+	return fmt.Errorf("block mode is not supported for Windows")
+}
+
+func (o *BlockOutput) BeginDirectory(ctx context.Context, relativePath string, e fs.Directory) error {
+	return fmt.Errorf("block mode is not supported for Windows")
+}

pkg/uploader/kopia/shim.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/content"
+	"github.com/kopia/kopia/repo/content/index"
 	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/repo/object"
 )
@@ -140,7 +141,7 @@ func (sr *shimRepository) Refresh(ctx context.Context) error {
 
 // ContentInfo not supported
 func (sr *shimRepository) ContentInfo(ctx context.Context, contentID content.ID) (content.Info, error) {
-	return nil, errors.New("ContentInfo is not supported")
+	return index.Info{}, errors.New("ContentInfo is not supported")
 }
 
 // PrefetchContents is not supported by unified repo

pkg/uploader/kopia/snapshot.go

@@ -243,10 +243,10 @@ func SnapshotSource(
 
 			mani, err := loadSnapshotFunc(ctx, rep, manifest.ID(parentSnapshot))
 			if err != nil {
-				return "", 0, errors.Wrapf(err, "Failed to load previous snapshot %v from kopia", parentSnapshot)
+				log.WithError(err).Warnf("Failed to load previous snapshot %v from kopia, fallback to full backup", parentSnapshot)
+			} else {
+				previous = append(previous, mani)
 			}
-
-			previous = append(previous, mani)
 		} else {
 			log.Infof("Searching for parent snapshot")
 

pkg/uploader/kopia/snapshot_test.go

@@ -114,7 +114,7 @@ func TestSnapshotSource(t *testing.T) {
 			notError: true,
 		},
 		{
-			name: "failed to load snapshot",
+			name: "failed to load snapshot, should fallback to full backup and not error",
 			args: []mockArgs{
 				{methodName: "LoadSnapshot", returns: []interface{}{manifest, errors.New("failed to load snapshot")}},
 				{methodName: "SaveSnapshot", returns: []interface{}{manifest.ID, nil}},
@@ -124,7 +124,7 @@ func TestSnapshotSource(t *testing.T) {
 				{methodName: "Upload", returns: []interface{}{manifest, nil}},
 				{methodName: "Flush", returns: []interface{}{nil}},
 			},
-			notError: false,
+			notError: true,
 		},
 		{
 			name: "failed to save snapshot",

pkg/uploader/provider/kopia_test.go

@@ -36,12 +36,12 @@ import (
 	"github.com/vmware-tanzu/velero/internal/credentials/mocks"
 	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
-	"github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/scheme"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 	udmrepo "github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
 	udmrepomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks"
 	"github.com/vmware-tanzu/velero/pkg/uploader"
 	"github.com/vmware-tanzu/velero/pkg/uploader/kopia"
+	"github.com/vmware-tanzu/velero/pkg/util"
 )
 
 type FakeBackupProgressUpdater struct {
@@ -65,7 +65,7 @@ func (f *FakeRestoreProgressUpdater) UpdateProgress(p *uploader.Progress) {}
 func TestRunBackup(t *testing.T) {
 	var kp kopiaProvider
 	kp.log = logrus.New()
-	updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: kp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(scheme.Scheme).Build()}
+	updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: kp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()}
 
 	testCases := []struct {
 		name           string
@@ -122,7 +122,7 @@ func TestRunBackup(t *testing.T) {
 func TestRunRestore(t *testing.T) {
 	var kp kopiaProvider
 	kp.log = logrus.New()
-	updater := FakeRestoreProgressUpdater{PodVolumeRestore: &velerov1api.PodVolumeRestore{}, Log: kp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(scheme.Scheme).Build()}
+	updater := FakeRestoreProgressUpdater{PodVolumeRestore: &velerov1api.PodVolumeRestore{}, Log: kp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()}
 
 	testCases := []struct {
 		name            string

pkg/uploader/provider/provider_test.go

@@ -28,7 +28,7 @@ import (
 	"github.com/vmware-tanzu/velero/internal/credentials"
 	"github.com/vmware-tanzu/velero/internal/credentials/mocks"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
-	"github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/scheme"
+	"github.com/vmware-tanzu/velero/pkg/util"
 )
 
 type NewUploaderProviderTestCase struct {
@@ -42,7 +42,7 @@ type NewUploaderProviderTestCase struct {
 func TestNewUploaderProvider(t *testing.T) {
 	// Mock objects or dependencies
 	ctx := context.Background()
-	client := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build()
+	client := fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()
 	repoIdentifier := "repoIdentifier"
 	bsl := &velerov1api.BackupStorageLocation{}
 	backupRepo := &velerov1api.BackupRepository{}

pkg/uploader/provider/restic_test.go

@@ -34,9 +34,9 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
-	"github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/scheme"
 	"github.com/vmware-tanzu/velero/pkg/restic"
 	"github.com/vmware-tanzu/velero/pkg/uploader"
+	"github.com/vmware-tanzu/velero/pkg/util"
 	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
 )
 
@@ -149,7 +149,7 @@ func TestResticRunBackup(t *testing.T) {
 				tc.volMode = uploader.PersistentVolumeFilesystem
 			}
 			if !tc.nilUpdater {
-				updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(scheme.Scheme).Build()}
+				updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()}
 				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, shared.UploaderConfig{}, &updater)
 			} else {
 				_, _, err = tc.rp.RunBackup(context.Background(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, shared.UploaderConfig{}, nil)
@@ -222,7 +222,7 @@ func TestResticRunRestore(t *testing.T) {
 			}
 			var err error
 			if !tc.nilUpdater {
-				updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(scheme.Scheme).Build()}
+				updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: context.Background(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()}
 				err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, &updater)
 			} else {
 				err = tc.rp.RunRestore(context.Background(), "", "var", tc.volMode, nil)

pkg/util/csi/util.go

@@ -0,0 +1,32 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csi
+
+import (
+	"strings"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/features"
+)
+
+const (
+	csiPluginNamePrefix = "velero.io/csi-"
+)
+
+func ShouldSkipAction(actionName string) bool {
+	return !features.IsEnabled(velerov1api.CSIFeatureFlag) && strings.Contains(actionName, csiPluginNamePrefix)
+}

pkg/util/csi/util_test.go

@@ -0,0 +1,36 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package csi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/vmware-tanzu/velero/pkg/features"
+)
+
+func TestCSIFeatureNotEnabledAndPluginIsFromCSI(t *testing.T) {
+
+	features.NewFeatureFlagSet("EnableCSI")
+	require.False(t, ShouldSkipAction("abc"))
+	require.False(t, ShouldSkipAction("velero.io/csi-pvc-backupper"))
+
+	features.NewFeatureFlagSet("")
+	require.True(t, ShouldSkipAction("velero.io/csi-pvc-backupper"))
+	require.False(t, ShouldSkipAction("abc"))
+}