Merge branch 'main' of https://github.com/qiuming-best/velero into perf-test

.github/workflows/pr-ci-check.yml

@@ -24,8 +24,9 @@ jobs:
       - name: Make ci
         run: make ci
       - name: Upload test coverage
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.out
           verbose: true
+          fail_ci_if_error: true

Dockerfile

@@ -41,7 +41,9 @@ COPY . /go/src/github.com/vmware-tanzu/velero
 RUN mkdir -p /output/usr/bin && \
     export GOARM=$( echo "${GOARM}" | cut -c2-) && \
     go build -o /output/${BIN} \
-    -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN}
+    -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN} && \
+    go build -o /output/velero-helper \
+    -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-helper
 
 # Restic binary build section
 FROM --platform=$BUILDPLATFORM golang:1.20-bullseye as restic-builder
@@ -52,7 +54,7 @@ ARG TARGETARCH
 ARG TARGETVARIANT
 ARG RESTIC_VERSION
 
-env CGO_ENABLED=0 \
+ENV CGO_ENABLED=0 \
     GO111MODULE=on \
     GOPROXY=${GOPROXY} \
     GOOS=${TARGETOS} \

changelogs/unreleased/6558-kaovilai

@@ -0,0 +1 @@
+Non default s3 credential profiles work on Unified Repository Provider (kopia)

changelogs/unreleased/6587-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue 6575, flush the repo after delete the snapshot, otherwise, the changes(deleting repo snapshot) cannot be committed to the repo.

changelogs/unreleased/6594-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue #6571, fix the problem for restore item operation to set the errors correctly so that they can be recorded by Velero restore and then reflect the correct status for Velero restore.

changelogs/unreleased/6598-aws_creds

@@ -0,0 +1 @@
+Fix how the AWS credentials are obtained from configuration

changelogs/unreleased/6634-27149chen

@@ -0,0 +1 @@
+Fixes #6498. Get resource client again after restore actions in case resource's gv is changed. This is an improvement of pr #6499, to support group changes. A group change usually happens in a restore plugin which is used for resource conversion: convert a resource from a not supported gv to a supported gv

cmd/velero-helper/velero-helper.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+)
+
+const (
+	// workingModePause indicates it is for general purpose to hold the pod under running state
+	workingModePause = "pause"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Fprintln(os.Stderr, "ERROR: at least one argument must be provided, the working mode")
+		os.Exit(1)
+	}
+
+	switch os.Args[1] {
+	case workingModePause:
+		time.Sleep(time.Duration(1<<63 - 1))
+	default:
+		fmt.Fprintln(os.Stderr, "ERROR: wrong working mode provided")
+		os.Exit(1)
+	}
+}

config/crd/v2alpha1/bases/velero.io_datauploads.yaml

@@ -79,7 +79,7 @@ spec:
                 nullable: true
                 properties:
                   snapshotClass:
-                    description: StorageClass is the name of the snapshot class that
+                    description: SnapshotClass is the name of the snapshot class that
                       the volume snapshot is created with
                     type: string
                   storageClass:

design/upload-progress.md

@@ -89,7 +89,7 @@ will be deleted and at that point any uploads still in progress should be aborte
 
 ### Uploading (new)
 The "Uploading" phase signifies that the main part of the backup, including snapshotting has completed successfully
-and and uploading is continuing.  In the event of an error during uploading, the phase will change to 
+and uploading is continuing.  In the event of an error during uploading, the phase will change to 
 UploadingPartialFailure.  On success, the phase changes to Completed.  The backup cannot be
 restored from when it is in the Uploading state.
 

design/volume-snapshot-data-movement/volume-snapshot-data-movement.md

@@ -245,7 +245,7 @@ spec:
                   of the CSI snapshot.
                 properties:
                   snapshotClass:
-                    description: StorageClass is the name of the snapshot class that
+                    description: SnapshotClass is the name of the snapshot class that
                       the volume snapshot is created with
                     type: string                
                   storageClass:

hack/update-1fmt.sh

@@ -35,7 +35,7 @@ fi
 
 files="$(find . -type f -name '*.go' -not -path './.go/*' -not -path './vendor/*' -not -path './site/*' -not -path '*/generated/*' -not -name 'zz_generated*' -not -path '*/mocks/*')"
 echo "${ACTION} gofmt"
-output=$(printf '%s\n' "${files}" | xargs gofmt "${MODE}" -s)
+output=$(gofmt "${MODE}" -s ${files})
 if [[ -n "${output}" ]]; then
   VERIFY_FMT_FAILED=1
   echo "${output}"
@@ -47,7 +47,7 @@ else
 fi
 
 echo "${ACTION} goimports"
-output=$(printf '%s\n' "${files}" | xargs goimports "${MODE}" -local github.com/vmware-tanzu/velero)
+output=$(goimports "${MODE}" -local github.com/vmware-tanzu/velero ${files})
 if [[ -n "${output}" ]]; then
   VERIFY_IMPORTS_FAILED=1
   echo "${output}"

pkg/apis/velero/v1/register.go

@@ -20,6 +20,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 // Resource gets a Velero GroupResource for a specified resource
@@ -59,6 +60,18 @@ func CustomResources() map[string]typeInfo {
 	}
 }
 
+// CustomResourceKinds returns a list of all custom resources kinds within the Velero
+func CustomResourceKinds() sets.String {
+	kinds := sets.NewString()
+
+	resources := CustomResources()
+	for kind := range resources {
+		kinds.Insert(kind)
+	}
+
+	return kinds
+}
+
 func addKnownTypes(scheme *runtime.Scheme) error {
 	for _, typeInfo := range CustomResources() {
 		scheme.AddKnownTypes(SchemeGroupVersion, typeInfo.ItemType, typeInfo.ItemListType)

pkg/apis/velero/v2alpha1/data_upload_types.go

@@ -76,7 +76,7 @@ type CSISnapshotSpec struct {
 	// StorageClass is the name of the storage class of the PVC that the volume snapshot is created from
 	StorageClass string `json:"storageClass"`
 
-	// StorageClass is the name of the snapshot class that the volume snapshot is created with
+	// SnapshotClass is the name of the snapshot class that the volume snapshot is created with
 	// +optional
 	SnapshotClass string `json:"snapshotClass"`
 }

pkg/apis/velero/v2alpha1/register.go

@@ -20,6 +20,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 // Resource gets a Velero GroupResource for a specified resource
@@ -50,6 +51,18 @@ func CustomResources() map[string]typeInfo {
 	}
 }
 
+// CustomResourceKinds returns a list of all custom resources kinds within the Velero
+func CustomResourceKinds() sets.String {
+	kinds := sets.NewString()
+
+	resources := CustomResources()
+	for kind := range resources {
+		kinds.Insert(kind)
+	}
+
+	return kinds
+}
+
 func addKnownTypes(scheme *runtime.Scheme) error {
 	for _, typeInfo := range CustomResources() {
 		scheme.AddKnownTypes(SchemeGroupVersion, typeInfo.ItemType, typeInfo.ItemListType)

pkg/backup/item_backupper.go

@@ -525,6 +525,11 @@ func (ib *itemBackupper) takePVSnapshot(obj runtime.Unstructured, log logrus.Fie
 	// After that, this warning can be removed.
 	if boolptr.IsSetToTrue(ib.backupRequest.Spec.SnapshotMoveData) {
 		log.Warnf("VolumeSnapshotter plugin doesn't support data movement.")
+
+		if features.IsEnabled(velerov1api.CSIFeatureFlag) && pv.Spec.CSI == nil {
+			log.Warn("Cannot use CSI data mover to handle PV, because PV doesn't contain CSI in spec.",
+				" Fall back to Velero native snapshot.")
+		}
 	}
 
 	if ib.backupRequest.ResPolicies != nil {

pkg/cmd/cli/restore/delete.go

@@ -123,7 +123,7 @@ func Run(o *cli.DeleteOptions) error {
 			errs = append(errs, errors.WithStack(err))
 			continue
 		}
-		fmt.Printf("Restore %q deleted\n", r.Name)
+		fmt.Printf("Request to delete restore %q submitted successfully.\nThe restore will be fully deleted after all associated data (restore files in object storage) are removed.\n", r.Name)
 	}
 	return kubeerrs.NewAggregate(errs)
 }

pkg/cmd/cli/restore/get.go

@@ -59,6 +59,13 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command {
 				cmd.CheckError(err)
 			}
 
+			// Append "(Deleting)" to phase if deletionTimestamp is marked.
+			for i := range restores.Items {
+				if !restores.Items[i].DeletionTimestamp.IsZero() {
+					restores.Items[i].Status.Phase += " (Deleting)"
+				}
+			}
+
 			if printed, err := output.PrintWithFormat(c, restores); printed || err != nil {
 				cmd.CheckError(err)
 				return

pkg/cmd/cli/uninstall/uninstall.go

@@ -25,23 +25,31 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
+	appsv1api "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/labels"
 	kubeerrs "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/client"
 	"github.com/vmware-tanzu/velero/pkg/cmd"
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli"
+	"github.com/vmware-tanzu/velero/pkg/controller"
 	"github.com/vmware-tanzu/velero/pkg/install"
+	kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
 )
 
+var gracefulDeletionMaximumDuration = 1 * time.Minute
+
 // uninstallOptions collects all the options for uninstalling Velero from a Kubernetes cluster.
 type uninstallOptions struct {
 	wait  bool // deprecated
@@ -117,7 +125,6 @@ func Run(ctx context.Context, kbClient kbclient.Client, namespace string) error
 	}
 
 	// CRDs
-
 	veleroLabelSelector := labels.SelectorFromSet(install.Labels())
 	opts := []kbclient.DeleteAllOfOption{
 		kbclient.InNamespace(namespace),
@@ -160,8 +167,7 @@ func Run(ctx context.Context, kbClient kbclient.Client, namespace string) error
 }
 
 func deleteNamespace(ctx context.Context, kbClient kbclient.Client, namespace string) error {
-	// delete resources with finalizer attached first to ensure finalizer can be handled by corresponding controller and resources can be deleted successfully before controller's pod is deleted.
-	// otherwise the process of deleting namespace will get stuck in deleting those resources forever.
+	// Deal with resources with attached finalizers to ensure proper handling of those finalizers.
 	if err := deleteResourcesWithFinalizer(ctx, kbClient, namespace); err != nil {
 		return errors.Wrap(err, "Fail to remove finalizer from restores")
 	}
@@ -211,58 +217,136 @@ func deleteNamespace(ctx context.Context, kbClient kbclient.Client, namespace st
 	return nil
 }
 
+// A few things needed to be noticed here:
+// 1. When we delete resources with attached finalizers, the corresponding controller will deal with the finalizer then resources can be deleted successfully.
+// So it is important to delete these resources before deleting the pod that runs that controller.
+// 2. The controller may encounter errors while handling the finalizer, in such case, the controller will keep trying until it succeeds.
+// So it is important to set a timeout, once the process exceed the timeout, we will forcedly delete the resources by removing the finalizer from them,
+// otherwise the deletion process may get stuck indefinitely.
+// 3. There is only restore finalizer supported as of v1.12. If any new finalizers are added in the future, the corresponding deletion logic can be
+// incorporated into this function.
 func deleteResourcesWithFinalizer(ctx context.Context, kbClient kbclient.Client, namespace string) error {
-	//check if restore crd exists
+	fmt.Println("Waiting for resource with attached finalizer to be deleted")
+	return deleteRestore(ctx, kbClient, namespace)
+}
+
+func deleteRestore(ctx context.Context, kbClient kbclient.Client, namespace string) error {
+	// Check if restore crd exists, if it does not exist, return immediately.
+	var err error
 	v1crd := &apiextv1.CustomResourceDefinition{}
 	key := kbclient.ObjectKey{Name: "restores.velero.io"}
-	if err := kbClient.Get(ctx, key, v1crd); err != nil {
+	if err = kbClient.Get(ctx, key, v1crd); err != nil {
 		if apierrors.IsNotFound(err) {
 			return nil
 		} else {
-			return err
+			return errors.Wrap(err, "Error getting restore crd")
 		}
 	}
 
-	// delete all the restores
-	restoreList := &velerov1api.RestoreList{}
-	if err := kbClient.List(ctx, restoreList, &kbclient.ListOptions{Namespace: namespace}); err != nil {
-		return err
+	// First attempt to gracefully delete all the restore within a specified time frame, If the process exceeds the timeout limit,
+	// it is likely that there may be errors during the finalization of restores. In such cases, we should proceed with forcefully deleting the restores.
+	err = gracefullyDeleteRestore(ctx, kbClient, namespace)
+	if err != nil && err != wait.ErrWaitTimeout {
+		return errors.Wrap(err, "Error deleting restores")
 	}
+	if err == wait.ErrWaitTimeout {
+		err = forcedlyDeleteRestore(ctx, kbClient, namespace)
+		if err != nil {
+			return errors.Wrap(err, "Error deleting restores")
+		}
+	}
 
+	return nil
+}
+
+func gracefullyDeleteRestore(ctx context.Context, kbClient kbclient.Client, namespace string) error {
+	var err error
+	restoreList := &velerov1api.RestoreList{}
+	if err = kbClient.List(ctx, restoreList, &kbclient.ListOptions{Namespace: namespace}); err != nil {
+		return errors.Wrap(err, "Error getting restores during graceful deletion")
+	}
+
 	for i := range restoreList.Items {
-		if err := kbClient.Delete(ctx, &restoreList.Items[i]); err != nil {
+		if err = kbClient.Delete(ctx, &restoreList.Items[i]); err != nil {
 			if apierrors.IsNotFound(err) {
 				continue
+			}
+			return errors.Wrap(err, "Error deleting restores during graceful deletion")
+		}
+	}
+
+	// Wait for the deletion of all the restores within a specified time frame
+	err = wait.PollImmediate(time.Second, gracefulDeletionMaximumDuration, func() (bool, error) {
+		restoreList := &velerov1api.RestoreList{}
+		if errList := kbClient.List(ctx, restoreList, &kbclient.ListOptions{Namespace: namespace}); errList != nil {
+			return false, errList
+		}
+
+		if len(restoreList.Items) > 0 {
+			fmt.Print(".")
+			return false, nil
+		} else {
+			return true, nil
+		}
+	})
+
+	return err
+}
+
+func forcedlyDeleteRestore(ctx context.Context, kbClient kbclient.Client, namespace string) error {
+	// Delete velero deployment first in case:
+	// 1. finalizers will be added back by restore controller after they are removed at next step;
+	// 2. new restores attached with finalizer will be created by restore controller after we remove all the restores' finalizer at next step;
+	deploy := &appsv1api.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "velero",
+			Name:      namespace,
+		},
+	}
+
+	err := kbClient.Delete(ctx, deploy)
+	if err != nil && !apierrors.IsNotFound(err) {
+		return errors.Wrap(err, "Error deleting velero deployment during force deletion")
+	}
+
+	ctxc, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	checkFunc := func() {
+		deploy := &appsv1api.Deployment{}
+		key := kbclient.ObjectKey{Namespace: namespace, Name: "velero"}
+
+		if err = kbClient.Get(ctxc, key, deploy); err != nil {
+			if apierrors.IsNotFound(err) {
+				err = nil
+			}
+			cancel()
+			return
+		}
+	}
+
+	// Wait until velero deployment are deleted.
+	wait.Until(checkFunc, 100*time.Millisecond, ctxc.Done())
+	if err != nil {
+		return errors.Wrap(err, "Error deleting velero deployment during force deletion")
+	}
+
+	// Remove all the restores' finalizer so they can be deleted during the deletion of velero namespace.
+	restoreList := &velerov1api.RestoreList{}
+	if err := kbClient.List(ctx, restoreList, &kbclient.ListOptions{Namespace: namespace}); err != nil {
+		return errors.Wrap(err, "Error getting restores during force deletion")
+	}
+
+	for i := range restoreList.Items {
+		if controllerutil.ContainsFinalizer(&restoreList.Items[i], controller.ExternalResourcesFinalizer) {
+			update := &restoreList.Items[i]
+			original := update.DeepCopy()
+			controllerutil.RemoveFinalizer(update, controller.ExternalResourcesFinalizer)
+			if err := kubeutil.PatchResource(original, update, kbClient); err != nil {
+				return errors.Wrap(err, "Error removing restore finalizer during force deletion")
 			}
-			return err
 		}
 	}
 
-	fmt.Println("Waiting for resource with finalizer attached to be deleted")
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	var err error
-	checkFunc := func() {
-		restoreList := &velerov1api.RestoreList{}
-		if err = kbClient.List(ctx, restoreList, &kbclient.ListOptions{Namespace: namespace}); err != nil {
-			cancel()
-			return
-		}
-
-		if len(restoreList.Items) > 0 {
-			fmt.Print(".")
-		} else {
-			cancel()
-			return
-		}
-	}
-
-	// wait until all the restores are deleted
-	wait.Until(checkFunc, 100*time.Millisecond, ctx.Done())
-	if err != nil {
-		return err
-	}
-
 	return nil
 }

pkg/cmd/server/server.go

@@ -475,35 +475,30 @@ func (s *server) initDiscoveryHelper() error {
 func (s *server) veleroResourcesExist() error {
 	s.logger.Info("Checking existence of Velero custom resource definitions")
 
-	var veleroGroupVersion *metav1.APIResourceList
-	for _, gv := range s.discoveryHelper.Resources() {
-		if gv.GroupVersion == velerov1api.SchemeGroupVersion.String() {
-			veleroGroupVersion = gv
-			break
+	// add more group versions whenever available
+	gvResources := map[string]sets.String{
+		velerov1api.SchemeGroupVersion.String():       velerov1api.CustomResourceKinds(),
+		velerov2alpha1api.SchemeGroupVersion.String(): velerov2alpha1api.CustomResourceKinds(),
+	}
+
+	for _, lst := range s.discoveryHelper.Resources() {
+		if resources, found := gvResources[lst.GroupVersion]; found {
+			for _, resource := range lst.APIResources {
+				s.logger.WithField("kind", resource.Kind).Info("Found custom resource")
+				resources.Delete(resource.Kind)
+			}
 		}
 	}
 
-	if veleroGroupVersion == nil {
-		return fmt.Errorf("velero API group %s not found. Apply examples/common/00-prereqs.yaml to create it", velerov1api.SchemeGroupVersion)
-	}
-
-	foundResources := sets.NewString()
-	for _, resource := range veleroGroupVersion.APIResources {
-		foundResources.Insert(resource.Kind)
-	}
-
 	var errs []error
-	for kind := range velerov1api.CustomResources() {
-		if foundResources.Has(kind) {
-			s.logger.WithField("kind", kind).Debug("Found custom resource")
-			continue
+	for gv, resources := range gvResources {
+		for kind := range resources {
+			errs = append(errs, errors.Errorf("custom resource %s not found in Velero API group %s", kind, gv))
 		}
-
-		errs = append(errs, errors.Errorf("custom resource %s not found in Velero API group %s", kind, velerov1api.SchemeGroupVersion))
 	}
 
 	if len(errs) > 0 {
-		errs = append(errs, errors.New("Velero custom resources not found - apply examples/common/00-prereqs.yaml to update the custom resource definitions"))
+		errs = append(errs, errors.New("Velero custom resources not found - apply config/crd/v1/bases/*.yaml,config/crd/v2alpha1/bases*.yaml, to update the custom resource definitions"))
 		return kubeerrs.NewAggregate(errs)
 	}
 
@@ -586,13 +581,6 @@ func (s *server) checkNodeAgent() {
 }
 
 func (s *server) initRepoManager() error {
-	// warn if node agent does not exist
-	if err := nodeagent.IsRunning(s.ctx, s.kubeClient, s.namespace); err == nodeagent.ErrDaemonSetNotFound {
-		s.logger.Warn("Velero node agent not found; pod volume backups/restores will not work until it's created")
-	} else if err != nil {
-		s.logger.WithError(errors.WithStack(err)).Warn("Error checking for existence of velero node agent")
-	}
-
 	// ensure the repo key secret is set up
 	if err := repokey.EnsureCommonRepositoryKey(s.kubeClient.CoreV1(), s.namespace); err != nil {
 		return err

pkg/cmd/server/server_test.go

@@ -32,8 +32,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
-	v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
 	"github.com/vmware-tanzu/velero/pkg/client/mocks"
 	"github.com/vmware-tanzu/velero/pkg/controller"
 	discovery_mocks "github.com/vmware-tanzu/velero/pkg/discovery/mocks"
@@ -64,24 +64,40 @@ func TestVeleroResourcesExist(t *testing.T) {
 	}
 	assert.Error(t, server.veleroResourcesExist())
 
-	// Velero API group doesn't contain any custom resources: should error
-	veleroAPIResourceList := &metav1.APIResourceList{
-		GroupVersion: v1.SchemeGroupVersion.String(),
+	// Velero v1 API group doesn't contain any custom resources: should error
+	veleroAPIResourceListVelerov1 := &metav1.APIResourceList{
+		GroupVersion: velerov1api.SchemeGroupVersion.String(),
 	}
 
-	fakeDiscoveryHelper.ResourceList = append(fakeDiscoveryHelper.ResourceList, veleroAPIResourceList)
+	fakeDiscoveryHelper.ResourceList = append(fakeDiscoveryHelper.ResourceList, veleroAPIResourceListVelerov1)
 	assert.Error(t, server.veleroResourcesExist())
 
-	// Velero API group contains all custom resources: should not error
-	for kind := range v1.CustomResources() {
-		veleroAPIResourceList.APIResources = append(veleroAPIResourceList.APIResources, metav1.APIResource{
+	// Velero v2alpha1 API group doesn't contain any custom resources: should error
+	veleroAPIResourceListVeleroV2alpha1 := &metav1.APIResourceList{
+		GroupVersion: velerov2alpha1api.SchemeGroupVersion.String(),
+	}
+
+	fakeDiscoveryHelper.ResourceList = append(fakeDiscoveryHelper.ResourceList, veleroAPIResourceListVeleroV2alpha1)
+	assert.Error(t, server.veleroResourcesExist())
+
+	// Velero v1 API group contains all custom resources, but v2alpha1 doesn't contain any custom resources: should error
+	for kind := range velerov1api.CustomResources() {
+		veleroAPIResourceListVelerov1.APIResources = append(veleroAPIResourceListVelerov1.APIResources, metav1.APIResource{
+			Kind: kind,
+		})
+	}
+	assert.Error(t, server.veleroResourcesExist())
+
+	// Velero v1 and v2alpha1 API group contain all custom resources: should not error
+	for kind := range velerov2alpha1api.CustomResources() {
+		veleroAPIResourceListVeleroV2alpha1.APIResources = append(veleroAPIResourceListVeleroV2alpha1.APIResources, metav1.APIResource{
 			Kind: kind,
 		})
 	}
 	assert.NoError(t, server.veleroResourcesExist())
 
 	// Velero API group contains some but not all custom resources: should error
-	veleroAPIResourceList.APIResources = veleroAPIResourceList.APIResources[:3]
+	veleroAPIResourceListVelerov1.APIResources = veleroAPIResourceListVelerov1.APIResources[:3]
 	assert.Error(t, server.veleroResourcesExist())
 }
 
@@ -270,6 +286,13 @@ func Test_veleroResourcesExist(t *testing.T) {
 				{Kind: "ServerStatusRequest"},
 			},
 		},
+		{
+			GroupVersion: velerov2alpha1api.SchemeGroupVersion.String(),
+			APIResources: []metav1.APIResource{
+				{Kind: "DataUpload"},
+				{Kind: "DataDownload"},
+			},
+		},
 	})
 	assert.Nil(t, server.veleroResourcesExist())
 }

pkg/cmd/util/output/restore_describer.go

@@ -45,6 +45,12 @@ func DescribeRestore(ctx context.Context, kbClient kbclient.Client, restore *vel
 			phase = velerov1api.RestorePhaseNew
 		}
 		phaseString := string(phase)
+
+		// Append "Deleting" to phaseString if deletionTimestamp is marked.
+		if !restore.DeletionTimestamp.IsZero() {
+			phaseString += " (Deleting)"
+		}
+
 		switch phase {
 		case velerov1api.RestorePhaseCompleted:
 			phaseString = color.GreenString(phaseString)

pkg/controller/backup_deletion_controller.go

@@ -34,7 +34,9 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	kubeerrs "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/utils/clock"
+
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/vmware-tanzu/velero/internal/credentials"
@@ -363,24 +365,45 @@ func (r *backupDeletionReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}); err != nil {
 		log.WithError(errors.WithStack(err)).Error("Error listing restore API objects")
 	} else {
+		// Restore files in object storage will be handled by restore finalizer, so we simply need to initiate a delete request on restores here.
 		for i, restore := range restoreList.Items {
 			if restore.Spec.BackupName != backup.Name {
 				continue
 			}
 			restoreLog := log.WithField("restore", kube.NamespaceAndName(&restoreList.Items[i]))
 
-			restoreLog.Info("Deleting restore log/results from backup storage")
-			if err := backupStore.DeleteRestore(restore.Name); err != nil {
-				errs = append(errs, err.Error())
-				// if we couldn't delete the restore files, don't delete the API object
-				continue
-			}
-
 			restoreLog.Info("Deleting restore referencing backup")
 			if err := r.Delete(ctx, &restoreList.Items[i]); err != nil {
 				errs = append(errs, errors.Wrapf(err, "error deleting restore %s", kube.NamespaceAndName(&restoreList.Items[i])).Error())
 			}
 		}
+
+		// Wait for the deletion of restores within certain amount of time.
+		// Notice that there could be potential errors during the finalization process, which may result in the failure to delete the restore.
+		// Therefore, it is advisable to set a timeout period for waiting.
+		err := wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
+			restoreList := &velerov1api.RestoreList{}
+			if err := r.List(ctx, restoreList, &client.ListOptions{Namespace: backup.Namespace, LabelSelector: selector}); err != nil {
+				return false, err
+			}
+			cnt := 0
+			for _, restore := range restoreList.Items {
+				if restore.Spec.BackupName != backup.Name {
+					continue
+				}
+				cnt++
+			}
+
+			if cnt > 0 {
+				return false, nil
+			} else {
+				return true, nil
+			}
+		})
+		if err != nil {
+			log.WithError(err).Error("Error polling for deletion of restores")
+			errs = append(errs, errors.Wrapf(err, "error deleting restore %s", err).Error())
+		}
 	}
 
 	if len(errs) == 0 {

pkg/controller/backup_deletion_controller_test.go

@@ -322,8 +322,6 @@ func TestBackupDeletionControllerReconcile(t *testing.T) {
 		td.backupStore.On("GetBackupVolumeSnapshots", input.Spec.BackupName).Return(snapshots, nil)
 		td.backupStore.On("GetBackupContents", input.Spec.BackupName).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil)
 		td.backupStore.On("DeleteBackup", input.Spec.BackupName).Return(nil)
-		td.backupStore.On("DeleteRestore", "restore-1").Return(nil)
-		td.backupStore.On("DeleteRestore", "restore-2").Return(nil)
 
 		_, err := td.controller.Reconcile(context.TODO(), td.req)
 		require.NoError(t, err)
@@ -363,8 +361,6 @@ func TestBackupDeletionControllerReconcile(t *testing.T) {
 		assert.Nil(t, err)
 
 		td.backupStore.AssertCalled(t, "DeleteBackup", input.Spec.BackupName)
-		td.backupStore.AssertCalled(t, "DeleteRestore", "restore-1")
-		td.backupStore.AssertCalled(t, "DeleteRestore", "restore-2")
 
 		// Make sure snapshot was deleted
 		assert.Equal(t, 0, td.volumeSnapshotter.SnapshotsTaken.Len())

pkg/controller/data_download_controller.go

@@ -255,7 +255,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
 }
 
 func (r *DataDownloadReconciler) runCancelableDataPath(ctx context.Context, fsRestore datapath.AsyncBR, dd *velerov2alpha1api.DataDownload, res *exposer.ExposeResult, log logrus.FieldLogger) (reconcile.Result, error) {
-	path, err := exposer.GetPodVolumeHostPath(ctx, res.ByPod.HostingPod, res.ByPod.PVC, r.client, r.fileSystem, log)
+	path, err := exposer.GetPodVolumeHostPath(ctx, res.ByPod.HostingPod, res.ByPod.VolumeName, r.client, r.fileSystem, log)
 	if err != nil {
 		return r.errorOut(ctx, dd, err, "error exposing host path for pod volume", log)
 	}

pkg/controller/data_download_controller_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -141,6 +142,18 @@ func initDataDownloadReconcilerWithError(objects []runtime.Object, needError ...
 }
 
 func TestDataDownloadReconcile(t *testing.T) {
+	daemonSet := &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "velero",
+			Name:      "node-agent",
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "DaemonSet",
+			APIVersion: appsv1.SchemeGroupVersion.String(),
+		},
+		Spec: appsv1.DaemonSetSpec{},
+	}
+
 	tests := []struct {
 		name              string
 		dd                *velerov2alpha1api.DataDownload
@@ -283,7 +296,7 @@ func TestDataDownloadReconcile(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			var objs []runtime.Object
 			if test.targetPVC != nil {
-				objs = []runtime.Object{test.targetPVC}
+				objs = []runtime.Object{test.targetPVC, daemonSet}
 			}
 			r, err := initDataDownloadReconciler(objs, test.needErrs...)
 			require.NoError(t, err)
@@ -330,7 +343,7 @@ func TestDataDownloadReconcile(t *testing.T) {
 						} else if test.notNilExpose {
 							hostingPod := builder.ForPod("test-ns", "test-name").Volumes(&corev1.Volume{Name: "test-pvc"}).Result()
 							hostingPod.ObjectMeta.SetUID("test-uid")
-							ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: hostingPod, PVC: "test-pvc"}}, nil)
+							ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: hostingPod, VolumeName: "test-pvc"}}, nil)
 						} else if test.isGetExposeErr {
 							ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("Error to get restore exposer"))
 						}

pkg/controller/data_upload_controller.go

@@ -250,7 +250,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 func (r *DataUploadReconciler) runCancelableDataUpload(ctx context.Context, fsBackup datapath.AsyncBR, du *velerov2alpha1api.DataUpload, res *exposer.ExposeResult, log logrus.FieldLogger) (reconcile.Result, error) {
 	log.Info("Run cancelable dataUpload")
-	path, err := exposer.GetPodVolumeHostPath(ctx, res.ByPod.HostingPod, res.ByPod.PVC, r.client, r.fileSystem, log)
+	path, err := exposer.GetPodVolumeHostPath(ctx, res.ByPod.HostingPod, res.ByPod.VolumeName, r.client, r.fileSystem, log)
 	if err != nil {
 		return r.errorOut(ctx, du, err, "error exposing host path for pod volume", log)
 	}

pkg/controller/data_upload_controller_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -145,6 +146,18 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci
 			RestoreSize: &restoreSize,
 		},
 	}
+	daemonSet := &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "velero",
+			Name:      "node-agent",
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "DaemonSet",
+			APIVersion: appsv1.SchemeGroupVersion.String(),
+		},
+		Spec: appsv1.DaemonSetSpec{},
+	}
+
 	now, err := time.Parse(time.RFC1123, time.RFC1123)
 	if err != nil {
 		return nil, err
@@ -176,7 +189,7 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci
 	}
 
 	fakeSnapshotClient := snapshotFake.NewSimpleClientset(vsObject, vscObj)
-	fakeKubeClient := clientgofake.NewSimpleClientset()
+	fakeKubeClient := clientgofake.NewSimpleClientset(daemonSet)
 	fakeFS := velerotest.NewFakeFileSystem()
 	pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", "", dataUploadName)
 	_, err = fakeFS.Create(pathGlob)
@@ -240,7 +253,7 @@ func (f *fakeSnapshotExposer) GetExposed(ctx context.Context, du corev1.ObjectRe
 	if err != nil {
 		return nil, err
 	}
-	return &exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: pod, PVC: dataUploadName}}, nil
+	return &exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: pod, VolumeName: dataUploadName}}, nil
 }
 
 func (f *fakeSnapshotExposer) CleanUp(context.Context, corev1.ObjectReference, string, string) {

pkg/controller/restore_controller.go

@@ -535,8 +535,8 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu
 	// Completed yet.
 	inProgressOperations, _, opsCompleted, opsFailed, errs := getRestoreItemOperationProgress(restoreReq.Restore, pluginManager, *restoreReq.GetItemOperationsList())
 	if len(errs) > 0 {
-		for err := range errs {
-			restoreLog.Error(err)
+		for _, err := range errs {
+			restoreErrors.Velero = append(restoreErrors.Velero, fmt.Sprintf("error from restore item operation: %v", err))
 		}
 	}
 

pkg/exposer/csi_snapshot.go

@@ -94,7 +94,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1.Obje
 
 	curLog.Info("Exposing CSI snapshot")
 
-	volumeSnapshot, err := csi.WaitVolumeSnapshotReady(ctx, e.csiSnapshotClient, csiExposeParam.SnapshotName, csiExposeParam.SourceNamespace, csiExposeParam.Timeout)
+	volumeSnapshot, err := csi.WaitVolumeSnapshotReady(ctx, e.csiSnapshotClient, csiExposeParam.SnapshotName, csiExposeParam.SourceNamespace, csiExposeParam.Timeout, curLog)
 	if err != nil {
 		return errors.Wrapf(err, "error wait volume snapshot ready")
 	}
@@ -218,7 +218,7 @@ func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1.
 
 	curLog.WithField("backup pvc", backupPVCName).Info("Backup PVC is bound")
 
-	return &ExposeResult{ByPod: ExposeByPod{HostingPod: pod, PVC: backupPVCName}}, nil
+	return &ExposeResult{ByPod: ExposeByPod{HostingPod: pod, VolumeName: pod.Spec.Volumes[0].Name}}, nil
 }
 
 func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1.ObjectReference, vsName string, sourceNamespace string) {
@@ -345,6 +345,14 @@ func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject co
 func (e *csiSnapshotExposer) createBackupPod(ctx context.Context, ownerObject corev1.ObjectReference, backupPVC *corev1.PersistentVolumeClaim, label map[string]string) (*corev1.Pod, error) {
 	podName := ownerObject.Name
 
+	volumeName := string(ownerObject.UID)
+	containerName := string(ownerObject.UID)
+
+	podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace)
+	if err != nil {
+		return nil, errors.Wrap(err, "error to get inherited pod info from node-agent")
+	}
+
 	var gracePeriod int64 = 0
 
 	pod := &corev1.Pod{
@@ -365,19 +373,20 @@ func (e *csiSnapshotExposer) createBackupPod(ctx context.Context, ownerObject co
 		Spec: corev1.PodSpec{
 			Containers: []corev1.Container{
 				{
-					Name:            podName,
-					Image:           "alpine:latest",
-					ImagePullPolicy: corev1.PullIfNotPresent,
-					Command:         []string{"sleep", "infinity"},
+					Name:            containerName,
+					Image:           podInfo.image,
+					ImagePullPolicy: corev1.PullNever,
+					Command:         []string{"/velero-helper", "pause"},
 					VolumeMounts: []corev1.VolumeMount{{
-						Name:      backupPVC.Name,
-						MountPath: "/" + backupPVC.Name,
+						Name:      volumeName,
+						MountPath: "/" + volumeName,
 					}},
 				},
 			},
+			ServiceAccountName:            podInfo.serviceAccount,
 			TerminationGracePeriodSeconds: &gracePeriod,
 			Volumes: []corev1.Volume{{
-				Name: backupPVC.Name,
+				Name: volumeName,
 				VolumeSource: corev1.VolumeSource{
 					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
 						ClaimName: backupPVC.Name,

pkg/exposer/csi_snapshot_test.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 	clientTesting "k8s.io/client-go/testing"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 
 	velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -82,6 +83,18 @@ func TestExpose(t *testing.T) {
 		},
 	}
 
+	daemonSet := &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "velero",
+			Name:      "node-agent",
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "DaemonSet",
+			APIVersion: appsv1.SchemeGroupVersion.String(),
+		},
+		Spec: appsv1.DaemonSetSpec{},
+	}
+
 	tests := []struct {
 		name              string
 		snapshotClientObj []runtime.Object
@@ -257,6 +270,9 @@ func TestExpose(t *testing.T) {
 				vsObject,
 				vscObj,
 			},
+			kubeClientObj: []runtime.Object{
+				daemonSet,
+			},
 			kubeReactors: []reactor{
 				{
 					verb:     "create",

pkg/exposer/generic_restore.go

@@ -143,7 +143,7 @@ func (e *genericRestoreExposer) GetExposed(ctx context.Context, ownerObject core
 
 	curLog.WithField("restore pvc", restorePVCName).Info("Restore PVC is bound")
 
-	return &ExposeResult{ByPod: ExposeByPod{HostingPod: pod, PVC: restorePVCName}}, nil
+	return &ExposeResult{ByPod: ExposeByPod{HostingPod: pod, VolumeName: pod.Spec.Volumes[0].Name}}, nil
 }
 
 func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1.ObjectReference) {
@@ -251,6 +251,14 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec
 	restorePodName := ownerObject.Name
 	restorePVCName := ownerObject.Name
 
+	volumeName := string(ownerObject.UID)
+	containerName := string(ownerObject.UID)
+
+	podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace)
+	if err != nil {
+		return nil, errors.Wrap(err, "error to get inherited pod info from node-agent")
+	}
+
 	var gracePeriod int64 = 0
 
 	pod := &corev1.Pod{
@@ -271,19 +279,20 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec
 		Spec: corev1.PodSpec{
 			Containers: []corev1.Container{
 				{
-					Name:            restorePodName,
-					Image:           "alpine:latest",
-					ImagePullPolicy: corev1.PullIfNotPresent,
-					Command:         []string{"sleep", "infinity"},
+					Name:            containerName,
+					Image:           podInfo.image,
+					ImagePullPolicy: corev1.PullNever,
+					Command:         []string{"/velero-helper", "pause"},
 					VolumeMounts: []corev1.VolumeMount{{
-						Name:      restorePVCName,
-						MountPath: "/" + restorePVCName,
+						Name:      volumeName,
+						MountPath: "/" + volumeName,
 					}},
 				},
 			},
+			ServiceAccountName:            podInfo.serviceAccount,
 			TerminationGracePeriodSeconds: &gracePeriod,
 			Volumes: []corev1.Volume{{
-				Name: restorePVCName,
+				Name: volumeName,
 				VolumeSource: corev1.VolumeSource{
 					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
 						ClaimName: restorePVCName,

pkg/exposer/generic_restore_test.go

@@ -30,6 +30,7 @@ import (
 	velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1api "k8s.io/api/core/v1"
 	clientTesting "k8s.io/client-go/testing"
 )
@@ -64,6 +65,18 @@ func TestRestoreExpose(t *testing.T) {
 		},
 	}
 
+	daemonSet := &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "velero",
+			Name:      "node-agent",
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "DaemonSet",
+			APIVersion: appsv1.SchemeGroupVersion.String(),
+		},
+		Spec: appsv1.DaemonSetSpec{},
+	}
+
 	tests := []struct {
 		name            string
 		kubeClientObj   []runtime.Object
@@ -97,6 +110,7 @@ func TestRestoreExpose(t *testing.T) {
 			ownerRestore:    restore,
 			kubeClientObj: []runtime.Object{
 				targetPVCObj,
+				daemonSet,
 			},
 			kubeReactors: []reactor{
 				{
@@ -116,6 +130,7 @@ func TestRestoreExpose(t *testing.T) {
 			ownerRestore:    restore,
 			kubeClientObj: []runtime.Object{
 				targetPVCObj,
+				daemonSet,
 			},
 			kubeReactors: []reactor{
 				{

pkg/exposer/host_path.go

@@ -34,23 +34,23 @@ var getVolumeDirectory = kube.GetVolumeDirectory
 var singlePathMatch = kube.SinglePathMatch
 
 // GetPodVolumeHostPath returns a path that can be accessed from the host for a given volume of a pod
-func GetPodVolumeHostPath(ctx context.Context, pod *corev1.Pod, pvcName string,
+func GetPodVolumeHostPath(ctx context.Context, pod *corev1.Pod, volumeName string,
 	cli ctrlclient.Client, fs filesystem.Interface, log logrus.FieldLogger) (datapath.AccessPoint, error) {
-	logger := log.WithField("pod name", pod.Name).WithField("pod UID", pod.GetUID()).WithField("pvc", pvcName)
+	logger := log.WithField("pod name", pod.Name).WithField("pod UID", pod.GetUID()).WithField("volume", volumeName)
 
-	volDir, err := getVolumeDirectory(ctx, logger, pod, pvcName, cli)
+	volDir, err := getVolumeDirectory(ctx, logger, pod, volumeName, cli)
 	if err != nil {
-		return datapath.AccessPoint{}, errors.Wrapf(err, "error getting volume directory name for pvc %s in pod %s", pvcName, pod.Name)
+		return datapath.AccessPoint{}, errors.Wrapf(err, "error getting volume directory name for volume %s in pod %s", volumeName, pod.Name)
 	}
 
-	logger.WithField("volDir", volDir).Info("Got volume for backup PVC")
+	logger.WithField("volDir", volDir).Info("Got volume dir")
 
 	pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(pod.GetUID()), volDir)
 	logger.WithField("pathGlob", pathGlob).Debug("Looking for path matching glob")
 
 	path, err := singlePathMatch(pathGlob, fs, logger)
 	if err != nil {
-		return datapath.AccessPoint{}, errors.Wrapf(err, "error identifying unique volume path on host for pvc %s in pod %s", pvcName, pod.Name)
+		return datapath.AccessPoint{}, errors.Wrapf(err, "error identifying unique volume path on host for volume %s in pod %s", volumeName, pod.Name)
 	}
 
 	logger.WithField("path", path).Info("Found path matching glob")

pkg/exposer/host_path_test.go

@@ -48,7 +48,7 @@ func TestGetPodVolumeHostPath(t *testing.T) {
 			},
 			pod: builder.ForPod(velerov1api.DefaultNamespace, "fake-pod-1").Result(),
 			pvc: "fake-pvc-1",
-			err: "error getting volume directory name for pvc fake-pvc-1 in pod fake-pod-1: fake-error-1",
+			err: "error getting volume directory name for volume fake-pvc-1 in pod fake-pod-1: fake-error-1",
 		},
 		{
 			name: "single path match fail",
@@ -60,7 +60,7 @@ func TestGetPodVolumeHostPath(t *testing.T) {
 			},
 			pod: builder.ForPod(velerov1api.DefaultNamespace, "fake-pod-2").Result(),
 			pvc: "fake-pvc-1",
-			err: "error identifying unique volume path on host for pvc fake-pvc-1 in pod fake-pod-2: fake-error-2",
+			err: "error identifying unique volume path on host for volume fake-pvc-1 in pod fake-pod-2: fake-error-2",
 		},
 	}
 

pkg/exposer/image.go

@@ -0,0 +1,49 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exposer
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/vmware-tanzu/velero/pkg/nodeagent"
+)
+
+type inheritedPodInfo struct {
+	image          string
+	serviceAccount string
+}
+
+func getInheritedPodInfo(ctx context.Context, client kubernetes.Interface, veleroNamespace string) (inheritedPodInfo, error) {
+	podInfo := inheritedPodInfo{}
+
+	podSpec, err := nodeagent.GetPodSpec(ctx, client, veleroNamespace)
+	if err != nil {
+		return podInfo, errors.Wrap(err, "error to get node-agent pod template")
+	}
+
+	if len(podSpec.Containers) != 1 {
+		return podInfo, errors.Wrap(err, "unexpected pod template from node-agent")
+	}
+
+	podInfo.image = podSpec.Containers[0].Image
+	podInfo.serviceAccount = podSpec.ServiceAccountName
+
+	return podInfo, nil
+}

pkg/exposer/types.go

@@ -33,5 +33,5 @@ type ExposeResult struct {
 // ExposeByPod defines the result for the expose method that a hosting pod is created
 type ExposeByPod struct {
 	HostingPod *corev1.Pod
-	PVC        string
+	VolumeName string
 }

pkg/nodeagent/node_agent.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/vmware-tanzu/velero/pkg/util/kube"
@@ -73,3 +74,12 @@ func IsRunningInNode(ctx context.Context, namespace string, nodeName string, pod
 
 	return errors.Errorf("daemonset pod not found in running state in node %s", nodeName)
 }
+
+func GetPodSpec(ctx context.Context, kubeClient kubernetes.Interface, namespace string) (*v1.PodSpec, error) {
+	ds, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, daemonSet, metav1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "error to get node-agent daemonset")
+	}
+
+	return &ds.Spec.Template.Spec, nil
+}

pkg/repository/config/aws.go

@@ -33,8 +33,13 @@ import (
 const (
 	// AWS specific environment variable
 	awsProfileEnvVar         = "AWS_PROFILE"
+	awsRoleEnvVar            = "AWS_ROLE_ARN"
+	awsKeyIDEnvVar           = "AWS_ACCESS_KEY_ID"
+	awsSecretKeyEnvVar       = "AWS_SECRET_ACCESS_KEY"
+	awsSessTokenEnvVar       = "AWS_SESSION_TOKEN"
 	awsProfileKey            = "profile"
 	awsCredentialsFileEnvVar = "AWS_SHARED_CREDENTIALS_FILE"
+	awsConfigFileEnvVar      = "AWS_CONFIG_FILE"
 )
 
 // GetS3ResticEnvVars gets the environment variables that restic
@@ -51,32 +56,46 @@ func GetS3ResticEnvVars(config map[string]string) (map[string]string, error) {
 		result[awsProfileEnvVar] = profile
 	}
 
+	// GetS3ResticEnvVars reads the AWS config, from files and envs
+	// if needed assumes the role and returns the session credentials
+	// setting these variables emulates what would happen for example when using kube2iam
+	if creds, err := GetS3Credentials(config); err == nil && creds != nil {
+		result[awsKeyIDEnvVar] = creds.AccessKeyID
+		result[awsSecretKeyEnvVar] = creds.SecretAccessKey
+		result[awsSessTokenEnvVar] = creds.SessionToken
+		result[awsCredentialsFileEnvVar] = ""
+		result[awsProfileEnvVar] = ""
+		result[awsConfigFileEnvVar] = ""
+	}
+
 	return result, nil
 }
 
 // GetS3Credentials gets the S3 credential values according to the information
 // of the provided config or the system's environment variables
 func GetS3Credentials(config map[string]string) (*credentials.Value, error) {
-	if len(os.Getenv("AWS_ROLE_ARN")) > 0 {
+	if os.Getenv(awsRoleEnvVar) != "" {
 		return nil, nil
 	}
 
+	opts := session.Options{}
 	credentialsFile := config[CredentialsFileKey]
 	if credentialsFile == "" {
 		credentialsFile = os.Getenv("AWS_SHARED_CREDENTIALS_FILE")
 	}
-
-	if credentialsFile == "" {
-		return nil, errors.New("missing credential file")
+	if credentialsFile != "" {
+		opts.SharedConfigFiles = append(opts.SharedConfigFiles, credentialsFile)
+		opts.SharedConfigState = session.SharedConfigEnable
 	}
 
-	creds := credentials.NewSharedCredentials(credentialsFile, "")
-	credValue, err := creds.Get()
+	sess, err := session.NewSessionWithOptions(opts)
 	if err != nil {
 		return nil, err
 	}
 
-	return &credValue, nil
+	creds, err := sess.Config.Credentials.Get()
+
+	return &creds, err
 }
 
 // GetAWSBucketRegion returns the AWS region that a bucket is in, or an error

pkg/repository/provider/unified_repo.go

@@ -67,7 +67,7 @@ const (
 	repoOpDescMaintain = "repo maintenance"
 	repoOpDescForget   = "forget"
 
-	repoConnectDesc = "unfied repo"
+	repoConnectDesc = "unified repo"
 )
 
 // NewUnifiedRepoProvider creates the service provider for Unified Repo
@@ -302,6 +302,11 @@ func (urp *unifiedRepoProvider) Forget(ctx context.Context, snapshotID string, p
 		return errors.Wrap(err, "error to delete manifest")
 	}
 
+	err = bkRepo.Flush(ctx)
+	if err != nil {
+		return errors.Wrap(err, "error to flush repo")
+	}
+
 	log.Debug("Forget snapshot complete")
 
 	return nil

pkg/repository/provider/unified_repo_test.go

@@ -783,6 +783,7 @@ func TestForget(t *testing.T) {
 		backupRepo      *reposervicenmocks.BackupRepo
 		retFuncOpen     []interface{}
 		retFuncDelete   interface{}
+		retFuncFlush    interface{}
 		credStoreReturn string
 		credStoreError  error
 		expectedErr     string
@@ -843,6 +844,37 @@ func TestForget(t *testing.T) {
 			},
 			expectedErr: "error to delete manifest: fake-error-3",
 		},
+		{
+			name:            "flush fail",
+			getter:          new(credmock.SecretStore),
+			credStoreReturn: "fake-password",
+			funcTable: localFuncTable{
+				getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
+					return map[string]string{}, nil
+				},
+				getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
+					return map[string]string{}, nil
+				},
+			},
+			repoService: new(reposervicenmocks.BackupRepoService),
+			backupRepo:  new(reposervicenmocks.BackupRepo),
+			retFuncOpen: []interface{}{
+				func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo {
+					return backupRepo
+				},
+
+				func(context.Context, udmrepo.RepoOptions) error {
+					return nil
+				},
+			},
+			retFuncDelete: func(context.Context, udmrepo.ID) error {
+				return nil
+			},
+			retFuncFlush: func(context.Context) error {
+				return errors.New("fake-error-4")
+			},
+			expectedErr: "error to flush repo: fake-error-4",
+		},
 	}
 
 	for _, tc := range testCases {
@@ -871,6 +903,7 @@ func TestForget(t *testing.T) {
 
 			if tc.backupRepo != nil {
 				backupRepo.On("DeleteManifest", mock.Anything, mock.Anything).Return(tc.retFuncDelete)
+				backupRepo.On("Flush", mock.Anything).Return(tc.retFuncFlush)
 				backupRepo.On("Close", mock.Anything).Return(nil)
 			}
 

pkg/restic/common.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -71,11 +72,26 @@ func TempCACertFile(caCert []byte, bsl string, fs filesystem.Interface) (string,
 	return name, nil
 }
 
+// environ is a slice of strings representing the environment, in the form "key=value".
+type environ []string
+
+// Unset a single environment variable.
+func (e *environ) Unset(key string) {
+	for i := range *e {
+		if strings.HasPrefix((*e)[i], key+"=") {
+			(*e)[i] = (*e)[len(*e)-1]
+			*e = (*e)[:len(*e)-1]
+			break
+		}
+	}
+}
+
 // CmdEnv returns a list of environment variables (in the format var=val) that
 // should be used when running a restic command for a particular backend provider.
 // This list is the current environment, plus any provider-specific variables restic needs.
 func CmdEnv(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) {
-	env := os.Environ()
+	var env environ
+	env = os.Environ()
 	customEnv := map[string]string{}
 	var err error
 
@@ -113,6 +129,10 @@ func CmdEnv(backupLocation *velerov1api.BackupStorageLocation, credentialFileSto
 	}
 
 	for k, v := range customEnv {
+		env.Unset(k)
+		if v == "" {
+			continue
+		}
 		env = append(env, fmt.Sprintf("%s=%s", k, v))
 	}
 

pkg/restore/restore.go

@@ -1366,6 +1366,15 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 		}
 	}
 
+	// The object apiVersion might get modified by a RestorePlugin so we need to
+	// get a new client to reflect updated resource path.
+	newGR := schema.GroupResource{Group: obj.GroupVersionKind().Group, Resource: groupResource.Resource}
+	resourceClient, err = ctx.getResourceClient(newGR, obj, obj.GetNamespace())
+	if err != nil {
+		errs.AddVeleroError(fmt.Errorf("error getting updated resource client for namespace %q, resource %q: %v", namespace, &groupResource, err))
+		return warnings, errs, itemExists
+	}
+
 	ctx.log.Infof("Attempting to restore %s: %v", obj.GroupVersionKind().Kind, name)
 	createdObj, restoreErr := resourceClient.Create(obj)
 	if restoreErr == nil {

pkg/uploader/kopia/progress_test.go

@@ -37,7 +37,7 @@ func TestThrottle_ShouldOutput(t *testing.T) {
 		expectedOutput bool
 	}{
 		{interval: time.Second, expectedOutput: true},
-		{interval: time.Second, throttle: time.Now().UnixNano() + int64(time.Nanosecond*10000), expectedOutput: false},
+		{interval: time.Second, throttle: time.Now().UnixNano() + int64(time.Nanosecond*100000000), expectedOutput: false},
 	}
 	p := new(Progress)
 	for _, tc := range testCases {

pkg/util/csi/volume_snapshot.go

@@ -26,9 +26,11 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
+	"github.com/vmware-tanzu/velero/pkg/util/stringptr"
 
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1"
@@ -44,8 +46,9 @@ const (
 
 // WaitVolumeSnapshotReady waits a VS to become ready to use until the timeout reaches
 func WaitVolumeSnapshotReady(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface,
-	volumeSnapshot string, volumeSnapshotNS string, timeout time.Duration) (*snapshotv1api.VolumeSnapshot, error) {
+	volumeSnapshot string, volumeSnapshotNS string, timeout time.Duration, log logrus.FieldLogger) (*snapshotv1api.VolumeSnapshot, error) {
 	var updated *snapshotv1api.VolumeSnapshot
+	errMessage := sets.NewString()
 
 	err := wait.PollImmediate(waitInternal, timeout, func() (bool, error) {
 		tmpVS, err := snapshotClient.VolumeSnapshots(volumeSnapshotNS).Get(ctx, volumeSnapshot, metav1.GetOptions{})
@@ -53,7 +56,15 @@ func WaitVolumeSnapshotReady(ctx context.Context, snapshotClient snapshotter.Sna
 			return false, errors.Wrapf(err, fmt.Sprintf("error to get volumesnapshot %s/%s", volumeSnapshotNS, volumeSnapshot))
 		}
 
-		if tmpVS.Status == nil || tmpVS.Status.BoundVolumeSnapshotContentName == nil || !boolptr.IsSetToTrue(tmpVS.Status.ReadyToUse) || tmpVS.Status.RestoreSize == nil {
+		if tmpVS.Status == nil {
+			return false, nil
+		}
+
+		if tmpVS.Status.Error != nil {
+			errMessage.Insert(stringptr.GetString(tmpVS.Status.Error.Message))
+		}
+
+		if !boolptr.IsSetToTrue(tmpVS.Status.ReadyToUse) {
 			return false, nil
 		}
 
@@ -61,6 +72,14 @@ func WaitVolumeSnapshotReady(ctx context.Context, snapshotClient snapshotter.Sna
 		return true, nil
 	})
 
+	if err == wait.ErrWaitTimeout {
+		err = errors.Errorf("volume snapshot is not ready until timeout, errors: %v", errMessage.List())
+	}
+
+	if errMessage.Len() > 0 {
+		log.Warnf("Some errors happened during waiting for ready snapshot, errors: %v", errMessage.List())
+	}
+
 	return updated, err
 }
 

pkg/util/csi/volume_snapshot_test.go

@@ -31,6 +31,7 @@ import (
 	clientTesting "k8s.io/client-go/testing"
 
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
+	"github.com/vmware-tanzu/velero/pkg/util/stringptr"
 
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
 )
@@ -55,6 +56,8 @@ func TestWaitVolumeSnapshotReady(t *testing.T) {
 		},
 	}
 
+	errMessage := "fake-snapshot-creation-error"
+
 	tests := []struct {
 		name      string
 		clientObj []runtime.Object
@@ -81,7 +84,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) {
 					},
 				},
 			},
-			err: "timed out waiting for the condition",
+			err: "volume snapshot is not ready until timeout, errors: []",
 		},
 		{
 			name:      "vsc is nil in status",
@@ -96,7 +99,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) {
 					Status: &snapshotv1api.VolumeSnapshotStatus{},
 				},
 			},
-			err: "timed out waiting for the condition",
+			err: "volume snapshot is not ready until timeout, errors: []",
 		},
 		{
 			name:      "ready to use is nil in status",
@@ -113,10 +116,10 @@ func TestWaitVolumeSnapshotReady(t *testing.T) {
 					},
 				},
 			},
-			err: "timed out waiting for the condition",
+			err: "volume snapshot is not ready until timeout, errors: []",
 		},
 		{
-			name:      "restore size is nil in status",
+			name:      "ready to use is false",
 			vsName:    "fake-vs",
 			namespace: "fake-ns",
 			clientObj: []runtime.Object{
@@ -127,11 +130,47 @@ func TestWaitVolumeSnapshotReady(t *testing.T) {
 					},
 					Status: &snapshotv1api.VolumeSnapshotStatus{
 						BoundVolumeSnapshotContentName: &vscName,
-						ReadyToUse:                     boolptr.True(),
+						ReadyToUse:                     boolptr.False(),
 					},
 				},
 			},
-			err: "timed out waiting for the condition",
+			err: "volume snapshot is not ready until timeout, errors: []",
+		},
+		{
+			name:      "snapshot creation error with message",
+			vsName:    "fake-vs",
+			namespace: "fake-ns",
+			clientObj: []runtime.Object{
+				&snapshotv1api.VolumeSnapshot{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "fake-vs",
+						Namespace: "fake-ns",
+					},
+					Status: &snapshotv1api.VolumeSnapshotStatus{
+						Error: &snapshotv1api.VolumeSnapshotError{
+							Message: &errMessage,
+						},
+					},
+				},
+			},
+			err: "volume snapshot is not ready until timeout, errors: [fake-snapshot-creation-error]",
+		},
+		{
+			name:      "snapshot creation error without message",
+			vsName:    "fake-vs",
+			namespace: "fake-ns",
+			clientObj: []runtime.Object{
+				&snapshotv1api.VolumeSnapshot{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "fake-vs",
+						Namespace: "fake-ns",
+					},
+					Status: &snapshotv1api.VolumeSnapshotStatus{
+						Error: &snapshotv1api.VolumeSnapshotError{},
+					},
+				},
+			},
+			err: "volume snapshot is not ready until timeout, errors: [" + stringptr.NilString + "]",
 		},
 		{
 			name:      "success",
@@ -148,7 +187,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...)
 
-			vs, err := WaitVolumeSnapshotReady(context.Background(), fakeSnapshotClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond)
+			vs, err := WaitVolumeSnapshotReady(context.Background(), fakeSnapshotClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond, velerotest.NewLogger())
 			if err != nil {
 				assert.EqualError(t, err, test.err)
 			} else {

pkg/util/stringptr/stringptr.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2017 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stringptr
+
+const NilString = "<nil>"
+
+func GetString(str *string) string {
+	if str == nil {
+		return NilString
+	} else {
+		return *str
+	}
+}

site/content/docs/main/restore-resource-modifiers.md

@@ -0,0 +1,103 @@
+---
+title: "Restore Resource Modifiers"
+layout: docs
+---
+
+## Resource Modifiers
+Velero provides a generic ability to modify the resources during restore by specifying json patches. The json patches are applied to the resources before they are restored. The json patches are specified in a configmap and the configmap is referenced in the restore command. 
+
+**Creating resource Modifiers**
+
+Below is the two-step of using resource modifiers to modify the resources during restore.
+1. Creating resource modifiers configmap
+
+   You need to create one configmap in Velero install namespace from a YAML file that defined resource modifiers. The creating command would be like the below:
+   ```bash
+   kubectl create cm <configmap-name> --from-file <yaml-file> -n velero
+   ```
+2. Creating a restore reference to the defined resource policies
+
+   You can create a restore with the flag `--resource-modifier-configmap`, which will apply the defined resource modifiers to the current restore. The creating command would be like the below:
+   ```bash
+   velero restore create --resource-modifier-configmap <configmap-name>
+   ```
+
+**YAML template**
+
+- Yaml template:
+```yaml
+version: v1
+resourceModifierRules:
+- conditions:
+     groupKind: persistentvolumeclaims
+     resourceNameRegex: "^mysql.*$"
+     namespaces:
+     - bar
+     - foo
+  patches:
+  - operation: replace
+    path: "/spec/storageClassName"
+    value: "premium"
+  - operation: remove
+    path: "/metadata/labels/test"
+ ```
+
+- The above configmap will apply the JSON Patch to all the PVCs in the namespaces bar and foo with name starting with mysql. The JSON Patch will replace the storageClassName with "premium" and remove the label "test" from the PVCs.
+- You can specify multiple JSON Patches for a particular resource. The patches will be applied in the order specified in the configmap. A subsequent patch is applied in order and if multiple patches are specified for the same path, the last patch will override the previous patches.
+- You can can specify multiple resourceModifierRules in the configmap. The rules will be applied in the order specified in the configmap. 
+
+### Operations supported by the JSON Patch RFC: 
+- add
+- remove
+- replace
+- move
+- copy
+- test (covered below)
+
+### Advanced scenarios
+#### **Conditional patches using test operation**
+ The `test` operation can be used to check if a particular value is present in the resource. If the value is present, the patch will be applied. If the value is not present, the patch will not be applied. This can be used to apply a patch only if a particular value is present in the resource. For example, if you wish to change the storage class of a PVC only if the PVC is using a particular storage class, you can use the following configmap.
+```yaml
+version: v1
+resourceModifierRules:
+- conditions:
+    groupKind: persistentvolumeclaims.storage.k8s.io
+    resourceNameRegex: ".*"
+    namespaces:
+    - bar
+    - foo
+  patches:
+  - operation: test
+    path: "/spec/storageClassName"
+    value: "premium"
+  - operation: replace
+    path: "/spec/storageClassName"
+    value: "standard"
+```
+
+#### **Other examples**
+```yaml
+version: v1
+resourceModifierRules:
+- conditions:
+    groupKind: deployments.apps
+    resourceNameRegex: "^test-.*$"
+    namespaces:
+    - bar
+    - foo
+  patches:
+    # Dealing with complex values by escaping the yaml
+  - operation: add
+    path: "/spec/template/spec/containers/0"
+    value: "{\"name\": \"nginx\", \"image\": \"nginx:1.14.2\", \"ports\": [{\"containerPort\": 80}]}"
+    # Copy Operator
+  - operation: copy
+    from: "/spec/template/spec/containers/0"
+    path: "/spec/template/spec/containers/1"
+```
+
+**Note:** 
+- The design and approach is inspired from [kubectl patch command](https://github.com/kubernetes/kubectl/blob/0a61782351a027411b8b45b1443ec3dceddef421/pkg/cmd/patch/patch.go#L102C2-L104C1)
+-  Update a container's image using a json patch with positional arrays
+kubectl patch pod valid-pod -type='json' -p='[{"op": "replace", "path": "/spec/containers/0/image", "value":"new image"}]'
+- Before creating the resource modifier yaml, you can try it out using kubectl patch command. The same commands should work as it is.

site/content/docs/main/upgrade-to-1.12.md

@@ -1,28 +1,28 @@
 ---
-title: "Upgrading to Velero 1.11"
+title: "Upgrading to Velero 1.12"
 layout: docs
 ---
 
 ## Prerequisites
 
-- Velero [v1.10.x][5] installed.
+- Velero [v1.11.x][5] installed.
 
-If you're not yet running at least Velero v1.6, see the following:
+If you're not yet running at least Velero v1.7, see the following:
 
-- [Upgrading to v1.6][1]
-- [Upgrading to v1.7][2]
-- [Upgrading to v1.8][3]
-- [Upgrading to v1.9][4]
-- [Upgrading to v1.10][5]
+- [Upgrading to v1.7][1]
+- [Upgrading to v1.8][2]
+- [Upgrading to v1.9][3]
+- [Upgrading to v1.10][4]
+- [Upgrading to v1.11][5]
 
 Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) to make sure your version of Kubernetes is supported by the new version of Velero.
 
 ## Instructions
 
-**Caution:** From Velero v1.10, except for using restic to do file-system level backup and restore, kopia is also been integrated, it could be upgraded from v1.10 to v1.11 directly, but it would be a little bit of difference when upgrading to v1.11 from a version lower than v1.10.0. 
+**Caution:** From Velero v1.10, except for using restic to do file-system level backup and restore, kopia is also been integrated, it could be upgraded from v1.10 or higher to v1.12 directly, but it would be a little bit of difference when upgrading to v1.12 from a version lower than v1.10.0. 
 
 ### Upgrade from version lower than v1.10.0
-1. Install the Velero v1.11 command-line interface (CLI) by following the [instructions here][0].
+1. Install the Velero v1.12 command-line interface (CLI) by following the [instructions here][0].
 
     Verify that you've properly installed it by running:
 
@@ -34,7 +34,7 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw
 
     ```bash
     Client:
-        Version: v1.11.0
+        Version: v1.12.0
         Git commit: <git SHA>
     ```
 
@@ -51,7 +51,7 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw
     ```bash
     # uploader_type value could be restic or kopia
     kubectl get deploy -n velero -ojson \
-    | sed "s#\"image\"\: \"velero\/velero\:v[0-9]*.[0-9]*.[0-9]\"#\"image\"\: \"velero\/velero\:v1.11.0\"#g" \
+    | sed "s#\"image\"\: \"velero\/velero\:v[0-9]*.[0-9]*.[0-9]\"#\"image\"\: \"velero\/velero\:v1.12.0\"#g" \
     | sed "s#\"server\",#\"server\",\"--uploader-type=$uploader_type\",#g" \
     | sed "s#default-volumes-to-restic#default-volumes-to-fs-backup#g" \
     | sed "s#default-restic-prune-frequency#default-repo-maintain-frequency#g" \
@@ -60,7 +60,7 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw
 
     # optional, if using the restic daemon set
     echo $(kubectl get ds -n velero restic -ojson) \
-    | sed "s#\"image\"\: \"velero\/velero\:v[0-9]*.[0-9]*.[0-9]\"#\"image\"\: \"velero\/velero\:v1.11.0\"#g" \
+    | sed "s#\"image\"\: \"velero\/velero\:v[0-9]*.[0-9]*.[0-9]\"#\"image\"\: \"velero\/velero\:v1.12.0\"#g" \
     | sed "s#\"name\"\: \"restic\"#\"name\"\: \"node-agent\"#g" \
     | sed "s#\[ \"restic\",#\[ \"node-agent\",#g" \
     | kubectl apply -f -
@@ -77,41 +77,41 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw
 
     ```bash
     Client:
-        Version: v1.11.0
+        Version: v1.12.0
         Git commit: <git SHA>
 
     Server:
-        Version: v1.11.0
+        Version: v1.12.0
     ```
 
-### Upgrade from v1.10
-If it's directly upgraded from v1.10, the other steps remain the same only except for step 3 above. The details as below:
+### Upgrade from v1.10 or higher
+If it's directly upgraded from v1.10 or higher, the other steps remain the same only except for step 3 above. The details as below:
 
-3. Update the container image used by the Velero deployment, plugin and, optionally, the node agent daemon set:
+1. Update the container image used by the Velero deployment, plugin and, optionally, the node agent daemon set:
     ```bash
    # set the container and image of the init container for plugin accordingly,
    # if you are using other plugin
     kubectl set image deployment/velero \
-        velero=velero/velero:v1.11.0 \
-        velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.7.0 \
+        velero=velero/velero:v1.12.0 \
+        velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.8.0 \
         --namespace velero
 
     # optional, if using the node agent daemonset
     kubectl set image daemonset/node-agent \
-        node-agent=velero/velero:v1.11.0 \
+        node-agent=velero/velero:v1.12.0 \
         --namespace velero
     ```
 
 ## Notes
-If upgraded from v1.9.x, there still remains some resources left over in the cluster and never used in v1.11.x, which could be deleted through kubectl and it is based on your desire:
+If upgraded from v1.9.x, there still remains some resources left over in the cluster and never used in v1.12.x, which could be deleted through kubectl and it is based on your desire:
 
     - resticrepository CRD and related CRs
     - velero-restic-credentials secret in velero install namespace
 
 
 [0]: basic-install.md#install-the-cli
-[1]: https://velero.io/docs/v1.6/upgrade-to-1.6
-[2]: https://velero.io/docs/v1.7/upgrade-to-1.7
-[3]: https://velero.io/docs/v1.8/upgrade-to-1.8
-[4]: https://velero.io/docs/v1.9/upgrade-to-1.9
-[5]: https://velero.io/docs/v1.10/upgrade-to-1.10
+[1]: https://velero.io/docs/v1.7/upgrade-to-1.7
+[2]: https://velero.io/docs/v1.8/upgrade-to-1.8
+[3]: https://velero.io/docs/v1.9/upgrade-to-1.9
+[4]: https://velero.io/docs/v1.10/upgrade-to-1.10
+[5]: https://velero.io/docs/v1.11/upgrade-to-1.11

site/data/docs/main-toc.yml

@@ -13,8 +13,8 @@ toc:
         url: /basic-install
       - page: Customize Installation
         url: /customize-installation
-      - page: Upgrade to 1.11
-        url: /upgrade-to-1.11
+      - page: Upgrade to 1.12
+        url: /upgrade-to-1.12
       - page: Supported providers
         url: /supported-providers
       - page: Evaluation install
@@ -43,6 +43,8 @@ toc:
         url: /restore-reference
       - page: Restore hooks
         url: /restore-hooks
+      - page: Restore Resource Modifiers
+        url: /restore-resource-modifiers
       - page: Run in any namespace
         url: /namespace
       - page: CSI Support

test/e2e/backup/backup.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"time"
 
 	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo"
@@ -68,7 +69,9 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 				DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
 			})
 			if veleroCfg.InstallVelero {
-				err = VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)
+				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+				defer ctxCancel()
+				err = VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)
 				Expect(err).To(Succeed())
 			}
 		}

test/e2e/backups/sync_backups.go

@@ -69,7 +69,9 @@ func BackupsSyncTest() {
 				DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero)
 			})
 			if VeleroCfg.InstallVelero {
-				Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
+				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+				defer ctxCancel()
+				Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
 			}
 		}
 

test/e2e/backups/ttl.go

@@ -80,11 +80,11 @@ func TTLTest() {
 			By("Clean backups after test", func() {
 				DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
 			})
-			if veleroCfg.InstallVelero {
-				Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
-			}
-			ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Minute)
+			ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 			defer ctxCancel()
+			if veleroCfg.InstallVelero {
+				Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
+			}
 			Expect(DeleteNamespace(ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
 		}
 	})

test/e2e/basic/api-group/enable_api_group_extentions.go

@@ -73,13 +73,15 @@ func APIExtensionsVersionsTest() {
 			})
 			if veleroCfg.InstallVelero {
 				By("Uninstall Velero and delete CRD ", func() {
+					ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+					defer ctxCancel()
 					Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
-					Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
+					Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 						veleroCfg.VeleroNamespace)).To(Succeed())
 					Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed())
 
 					Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
-					Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
+					Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 						veleroCfg.VeleroNamespace)).To(Succeed())
 					Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed())
 				})

test/e2e/basic/api-group/enable_api_group_versions.go

@@ -45,13 +45,11 @@ var veleroCfg VeleroConfig
 
 type apiGropuVersionsTest struct {
 	name       string
-	namespaces []string
 	srcCrdYaml string
 	srcCRs     map[string]string
 	tgtCrdYaml string
 	tgtVer     string
 	cm         *corev1api.ConfigMap
-	gvs        map[string][]string
 	want       map[string]map[string]string
 }
 
@@ -108,6 +106,8 @@ func APIGropuVersionsTest() {
 
 	Context("When EnableAPIGroupVersions flag is set", func() {
 		It("Should back up API group version and restore by version priority", func() {
+			ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
+			defer ctxCancel()
 			Expect(runEnableAPIGroupVersionsTests(
 				ctx,
 				*veleroCfg.ClientToInstallVelero,
@@ -121,12 +121,12 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, grou
 	tests := []apiGropuVersionsTest{
 		{
 			name:       "Target and source cluster preferred versions match; Preferred version v1 is restored (Priority 1, Case A).",
-			srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
+			srcCrdYaml: "../testdata/enable_api_group_versions/case-a-source.yaml",
 			srcCRs: map[string]string{
-				"v1":       "testdata/enable_api_group_versions/music_v1_rockband.yaml",
-				"v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
+				"v1":       "../testdata/enable_api_group_versions/music_v1_rockband.yaml",
+				"v1alpha1": "../testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
 			},
-			tgtCrdYaml: "testdata/enable_api_group_versions/case-a-target.yaml",
+			tgtCrdYaml: "../testdata/enable_api_group_versions/case-a-target.yaml",
 			tgtVer:     "v1",
 			cm:         nil,
 			want: map[string]map[string]string{
@@ -140,13 +140,13 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, grou
 		},
 		{
 			name:       "Latest common non-preferred supported version v2beta2 is restored (Priority 3, Case D).",
-			srcCrdYaml: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
+			srcCrdYaml: "../testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
 			srcCRs: map[string]string{
-				"v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
-				"v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
-				"v1":      "testdata/enable_api_group_versions/music_v1_rockband.yaml",
+				"v2beta2": "../testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
+				"v2beta1": "../testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
+				"v1":      "../testdata/enable_api_group_versions/music_v1_rockband.yaml",
 			},
-			tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
+			tgtCrdYaml: "../testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
 			tgtVer:     "v2beta2",
 			cm:         nil,
 			want: map[string]map[string]string{
@@ -160,25 +160,25 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, grou
 		},
 		{
 			name:       "No common supported versions means no rockbands custom resource is restored.",
-			srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
+			srcCrdYaml: "../testdata/enable_api_group_versions/case-a-source.yaml",
 			srcCRs: map[string]string{
-				"v1":       "testdata/enable_api_group_versions/music_v1_rockband.yaml",
-				"v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
+				"v1":       "../testdata/enable_api_group_versions/music_v1_rockband.yaml",
+				"v1alpha1": "../testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
 			},
-			tgtCrdYaml: "testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml",
+			tgtCrdYaml: "../testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml",
 			tgtVer:     "",
 			cm:         nil,
 			want:       nil,
 		},
 		{
 			name:       "User config map overrides Priority 3, Case D and restores v2beta1",
-			srcCrdYaml: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
+			srcCrdYaml: "../testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
 			srcCRs: map[string]string{
-				"v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
-				"v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
-				"v1":      "testdata/enable_api_group_versions/music_v1_rockband.yaml",
+				"v2beta2": "../testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
+				"v2beta1": "../testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
+				"v1":      "../testdata/enable_api_group_versions/music_v1_rockband.yaml",
 			},
-			tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
+			tgtCrdYaml: "../testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
 			tgtVer:     "v2beta1",
 			cm: builder.ForConfigMap(veleroCfg.VeleroNamespace, "enableapigroupversions").Data(
 				"restoreResourcesVersionPriority",
@@ -195,9 +195,9 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, grou
 		},
 		{
 			name:       "Restore successful when CRD doesn't (yet) exist in target",
-			srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
+			srcCrdYaml: "../testdata/enable_api_group_versions/case-a-source.yaml",
 			srcCRs: map[string]string{
-				"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
+				"v1": "../testdata/enable_api_group_versions/music_v1_rockband.yaml",
 			},
 			tgtCrdYaml: "",
 			tgtVer:     "v1",

test/e2e/basic/pvc-selected-node-changing.go

@@ -91,7 +91,7 @@ func (p *PVCSelectedNodeChanging) CreateResources() error {
 	By("Prepare ConfigMap data", func() {
 		nodeNameList, err := GetWorkerNodes(p.Ctx)
 		Expect(err).To(Succeed())
-		Expect(len(nodeNameList) > 2).To(Equal(true))
+		Expect(len(nodeNameList) >= 2).To(Equal(true))
 		for _, nodeName := range nodeNameList {
 			if nodeName != p.oldNodeName {
 				p.newNodeName = nodeName

test/e2e/e2e_suite_test.go

@@ -22,6 +22,7 @@ import (
 	"flag"
 	"fmt"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/reporters"
@@ -205,6 +206,8 @@ var _ = BeforeSuite(func() {
 var _ = AfterSuite(func() {
 	if VeleroCfg.InstallVelero && !VeleroCfg.Debug {
 		By("release test resources after testing")
-		Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
+		ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+		defer ctxCancel()
+		Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
 	}
 })

test/e2e/migration/migration.go

@@ -73,7 +73,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 		// need to uninstall Velero first in case of the affection of the existing global velero installation
 		if veleroCfg.InstallVelero {
 			By("Uninstall Velero", func() {
-				Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
+				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+				defer ctxCancel()
+				Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 					veleroCfg.VeleroNamespace)).To(Succeed())
 			})
 		}
@@ -86,13 +88,15 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 			// })
 			if veleroCfg.InstallVelero {
 				By(fmt.Sprintf("Uninstall Velero and delete sample workload namespace %s", migrationNamespace), func() {
+					ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+					defer ctxCancel()
 					Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
-					Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
+					Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 						veleroCfg.VeleroNamespace)).To(Succeed())
 					DeleteNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace, true)
 
 					Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
-					Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
+					Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 						veleroCfg.VeleroNamespace)).To(Succeed())
 					DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
 				})

test/e2e/upgrade/upgrade.go

@@ -77,7 +77,9 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 		// need to uninstall Velero first in case of the affection of the existing global velero installation
 		if veleroCfg.InstallVelero {
 			By("Uninstall Velero", func() {
-				Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
+				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+				defer ctxCancel()
+				Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 					veleroCfg.VeleroNamespace)).To(Succeed())
 			})
 		}
@@ -92,7 +94,9 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 			})
 			if veleroCfg.InstallVelero {
 				By("Uninstall Velero", func() {
-					Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
+					ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
+					defer ctxCancel()
+					Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 						veleroCfg.VeleroNamespace)).To(Succeed())
 				})
 			}

test/util/velero/install.go

@@ -127,6 +127,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 		RestoreHelperImage:     veleroCfg.RestoreHelperImage,
 		VeleroServerDebugMode:  veleroCfg.VeleroServerDebugMode,
 	})
+
 	if err != nil {
 		RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", "")
 		return errors.WithMessagef(err, "Failed to install Velero in the cluster")