Merge pull request #5192 from qiuming-best/release-1.9

Amend changelogs for v1.9.1

.goreleaser.yml

@@ -14,7 +14,7 @@
 
 dist: _output
 builds:
-  - main: ./cmd/velero/main.go
+  - main: ./cmd/velero/velero.go
     env:
       - CGO_ENABLED=0
     goos:

changelogs/CHANGELOG-1.9.md

@@ -1,3 +1,28 @@
+## v1.9.1
+### 2022-08-03
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.9.1
+
+### Container Image
+`velero/velero:v1.9.1`
+
+### Documentation
+https://velero.io/docs/v1.9/
+
+### Upgrading
+https://velero.io/docs/v1.9/upgrade-to-1.9/
+
+### All changes
+
+  * Fix bsl validation bug: the BSL is validated continually and doesn't respect the validation period configured (#5112, @ywk253100)
+  * Modify BackupStoreGetter to avoid BSL spec changes (#5134, @sseago)
+  * Delay CA file deletion in PVB controller. (#5150, @jxun)
+  * Skip registering "crd-remap-version" plugin when feature flag "EnableAPIGroupVersions" is set (#5173, @reasonerjt)
+  * Fix restic backups to multiple backup storage locations bug (#5175, @qiuming-best)
+  * Make CSI snapshot creation timeout configurable. (#5189, @jxun)
+  * Add annotation "pv.kubernetes.io/migrated-to" for CSI checking. (#5186, @jxun)
+
 ## v1.9.0
 ### 2022-06-13
 
@@ -102,3 +127,4 @@ With bumping up the API to v1 in CSI plugin, the v0.3.0 CSI plugin will only wor
   * Fix E2E test [Backups][Deletion][Restic] on GCP. (#4968, @jxun)
   * Disable status as sub resource in CRDs (#4972, @ywk253100)
   * Add more information for failing to get path or snapshot in restic backup and restore. (#4988, @jxun)
+  * When spec.RestoreStatus is empty, don't restore status (#5015, @sseago)

config/crd/v1/bases/velero.io_backups.yaml

@@ -37,6 +37,11 @@ spec:
           spec:
             description: BackupSpec defines the specification for a Velero backup.
             properties:
+              csiSnapshotTimeout:
+                description: CSISnapshotTimeout specifies the time used to wait for
+                  CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+                  returning error as timeout. The default value is 10 minute.
+                type: string
               defaultVolumesToRestic:
                 description: DefaultVolumesToRestic specifies whether restic should
                   be used to take a backup of all pod volumes by default.

config/crd/v1/bases/velero.io_schedules.yaml

@@ -61,6 +61,11 @@ spec:
                 description: Template is the definition of the Backup to be run on
                   the provided schedule
                 properties:
+                  csiSnapshotTimeout:
+                    description: CSISnapshotTimeout specifies the time used to wait
+                      for CSI VolumeSnapshot status turns to ReadyToUse during creation,
+                      before returning error as timeout. The default value is 10 minute.
+                    type: string
                   defaultVolumesToRestic:
                     description: DefaultVolumesToRestic specifies whether restic should
                       be used to take a backup of all pod volumes by default.

pkg/apis/velero/v1/backup.go

@@ -110,6 +110,12 @@ type BackupSpec struct {
 	// +optional
 	// +nullable
 	OrderedResources map[string]string `json:"orderedResources,omitempty"`
+
+	// CSISnapshotTimeout specifies the time used to wait for CSI VolumeSnapshot status turns to
+	// ReadyToUse during creation, before returning error as timeout.
+	// The default value is 10 minute.
+	// +optional
+	CSISnapshotTimeout metav1.Duration `json:"csiSnapshotTimeout,omitempty"`
 }
 
 // BackupHooks contains custom behaviors that should be executed at different phases of the backup.

pkg/apis/velero/v1/zz_generated.deepcopy.go

@@ -250,6 +250,7 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) {
 			(*out)[key] = val
 		}
 	}
+	out.CSISnapshotTimeout = in.CSISnapshotTimeout
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec.

pkg/builder/backup_builder.go

@@ -233,3 +233,9 @@ func (b *BackupBuilder) OrderedResources(orders map[string]string) *BackupBuilde
 	b.object.Spec.OrderedResources = orders
 	return b
 }
+
+// CSISnapshotTimeout sets the Backup's CSISnapshotTimeout
+func (b *BackupBuilder) CSISnapshotTimeout(timeout time.Duration) *BackupBuilder {
+	b.object.Spec.CSISnapshotTimeout.Duration = timeout
+	return b
+}

pkg/cmd/cli/backup/create.go

@@ -98,6 +98,7 @@ type CreateOptions struct {
 	SnapshotLocations       []string
 	FromSchedule            string
 	OrderedResources        string
+	CSISnapshotTimeout      time.Duration
 
 	client veleroclient.Interface
 }
@@ -122,6 +123,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringSliceVar(&o.SnapshotLocations, "volume-snapshot-locations", o.SnapshotLocations, "List of locations (at most one per provider) where volume snapshots should be stored.")
 	flags.VarP(&o.Selector, "selector", "l", "Only back up resources matching this label selector.")
 	flags.StringVar(&o.OrderedResources, "ordered-resources", "", "Mapping Kinds to an ordered list of specific resources of that Kind.  Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon.  Example: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'.  Optional.")
+	flags.DurationVar(&o.CSISnapshotTimeout, "csi-snapshot-timeout", o.CSISnapshotTimeout, "How long to wait for CSI snapshot creation before timeout.")
 	f := flags.VarPF(&o.SnapshotVolumes, "snapshot-volumes", "", "Take snapshots of PersistentVolumes as part of the backup.")
 	// this allows the user to just specify "--snapshot-volumes" as shorthand for "--snapshot-volumes=true"
 	// like a normal bool flag
@@ -332,7 +334,8 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro
 			LabelSelector(o.Selector.LabelSelector).
 			TTL(o.TTL).
 			StorageLocation(o.StorageLocation).
-			VolumeSnapshotLocations(o.SnapshotLocations...)
+			VolumeSnapshotLocations(o.SnapshotLocations...).
+			CSISnapshotTimeout(o.CSISnapshotTimeout)
 		if len(o.OrderedResources) > 0 {
 			orders, err := ParseOrderedResources(o.OrderedResources)
 			if err != nil {

pkg/cmd/cli/backup/create_test.go

@@ -19,6 +19,7 @@ package backup
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,6 +36,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) {
 	o.Labels.Set("velero.io/test=true")
 	o.OrderedResources = "pods=p1,p2;persistentvolumeclaims=pvc1,pvc2"
 	orders, err := ParseOrderedResources(o.OrderedResources)
+	o.CSISnapshotTimeout = 20 * time.Minute
 	assert.NoError(t, err)
 
 	backup, err := o.BuildBackup(testNamespace)
@@ -46,6 +48,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) {
 		SnapshotVolumes:         o.SnapshotVolumes.Value,
 		IncludeClusterResources: o.IncludeClusterResources.Value,
 		OrderedResources:        orders,
+		CSISnapshotTimeout:      metav1.Duration{Duration: o.CSISnapshotTimeout},
 	}, backup.Spec)
 
 	assert.Equal(t, map[string]string{

pkg/cmd/cli/schedule/create.go

@@ -145,6 +145,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 				VolumeSnapshotLocations: o.BackupOptions.SnapshotLocations,
 				DefaultVolumesToRestic:  o.BackupOptions.DefaultVolumesToRestic.Value,
 				OrderedResources:        orders,
+				CSISnapshotTimeout:      metav1.Duration{Duration: o.BackupOptions.CSISnapshotTimeout},
 			},
 			Schedule:                   o.Schedule,
 			UseOwnerReferencesInBackup: &o.UseOwnerReferencesInBackup,

pkg/cmd/server/plugin/plugin.go

@@ -19,9 +19,11 @@ package plugin
 import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/features"
+
 	"github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/client"
 	velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery"
@@ -36,11 +38,10 @@ func NewCommand(f client.Factory) *cobra.Command {
 		Hidden: true,
 		Short:  "INTERNAL COMMAND ONLY - not intended to be run directly by users",
 		Run: func(c *cobra.Command, args []string) {
-			pluginServer.
+			pluginServer = pluginServer.
 				RegisterBackupItemAction("velero.io/pv", newPVBackupItemAction).
 				RegisterBackupItemAction("velero.io/pod", newPodBackupItemAction).
 				RegisterBackupItemAction("velero.io/service-account", newServiceAccountBackupItemAction(f)).
-				RegisterBackupItemAction("velero.io/crd-remap-version", newRemapCRDVersionAction(f)).
 				RegisterRestoreItemAction("velero.io/job", newJobRestoreItemAction).
 				RegisterRestoreItemAction("velero.io/pod", newPodRestoreItemAction).
 				RegisterRestoreItemAction("velero.io/restic", newResticRestoreItemAction(f)).
@@ -55,13 +56,15 @@ func NewCommand(f client.Factory) *cobra.Command {
 				RegisterRestoreItemAction("velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction).
 				RegisterRestoreItemAction("velero.io/change-pvc-node-selector", newChangePVCNodeSelectorItemAction(f)).
 				RegisterRestoreItemAction("velero.io/apiservice", newAPIServiceRestoreItemAction).
-				RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction).
-				Serve()
+				RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction)
+			if !features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag) {
+				// Do not register crd-remap-version BIA if the API Group feature flag is enabled, so that the v1 CRD can be backed up
+				pluginServer = pluginServer.RegisterBackupItemAction("velero.io/crd-remap-version", newRemapCRDVersionAction(f))
+			}
+			pluginServer.Serve()
 		},
 	}
-
 	pluginServer.BindFlags(c.Flags())
-
 	return c
 }
 

pkg/cmd/server/server.go

@@ -102,6 +102,8 @@ const (
 	// the default TTL for a backup
 	defaultBackupTTL = 30 * 24 * time.Hour
 
+	defaultCSISnapshotTimeout = 10 * time.Minute
+
 	// defaultCredentialsDirectory is the path on disk where credential
 	// files will be written to
 	defaultCredentialsDirectory = "/tmp/credentials"
@@ -111,7 +113,7 @@ type serverConfig struct {
 	// TODO(2.0) Deprecate defaultBackupLocation
 	pluginDir, metricsAddress, defaultBackupLocation                        string
 	backupSyncPeriod, podVolumeOperationTimeout, resourceTerminatingTimeout time.Duration
-	defaultBackupTTL, storeValidationFrequency                              time.Duration
+	defaultBackupTTL, storeValidationFrequency, defaultCSISnapshotTimeout   time.Duration
 	restoreResourcePriorities                                               []string
 	defaultVolumeSnapshotLocations                                          map[string]string
 	restoreOnly                                                             bool
@@ -142,6 +144,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 			defaultVolumeSnapshotLocations:    make(map[string]string),
 			backupSyncPeriod:                  defaultBackupSyncPeriod,
 			defaultBackupTTL:                  defaultBackupTTL,
+			defaultCSISnapshotTimeout:         defaultCSISnapshotTimeout,
 			storeValidationFrequency:          defaultStoreValidationFrequency,
 			podVolumeOperationTimeout:         defaultPodVolumeOperationTimeout,
 			restoreResourcePriorities:         defaultRestorePriorities,
@@ -650,6 +653,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			s.config.defaultBackupLocation,
 			s.config.defaultVolumesToRestic,
 			s.config.defaultBackupTTL,
+			s.config.defaultCSISnapshotTimeout,
 			s.sharedInformerFactory.Velero().V1().VolumeSnapshotLocations().Lister(),
 			defaultVolumeSnapshotLocations,
 			s.metrics,

pkg/controller/backup_controller.go

@@ -87,6 +87,7 @@ type backupController struct {
 	defaultBackupLocation       string
 	defaultVolumesToRestic      bool
 	defaultBackupTTL            time.Duration
+	defaultCSISnapshotTimeout   time.Duration
 	snapshotLocationLister      velerov1listers.VolumeSnapshotLocationLister
 	defaultSnapshotLocations    map[string]string
 	metrics                     *metrics.ServerMetrics
@@ -111,6 +112,7 @@ func NewBackupController(
 	defaultBackupLocation string,
 	defaultVolumesToRestic bool,
 	defaultBackupTTL time.Duration,
+	defaultCSISnapshotTimeout time.Duration,
 	volumeSnapshotLocationLister velerov1listers.VolumeSnapshotLocationLister,
 	defaultSnapshotLocations map[string]string,
 	metrics *metrics.ServerMetrics,
@@ -135,6 +137,7 @@ func NewBackupController(
 		defaultBackupLocation:       defaultBackupLocation,
 		defaultVolumesToRestic:      defaultVolumesToRestic,
 		defaultBackupTTL:            defaultBackupTTL,
+		defaultCSISnapshotTimeout:   defaultCSISnapshotTimeout,
 		snapshotLocationLister:      volumeSnapshotLocationLister,
 		defaultSnapshotLocations:    defaultSnapshotLocations,
 		metrics:                     metrics,
@@ -359,6 +362,11 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup) *pkg
 		request.Spec.TTL.Duration = c.defaultBackupTTL
 	}
 
+	if request.Spec.CSISnapshotTimeout.Duration == 0 {
+		// set default CSI VolumeSnapshot timeout
+		request.Spec.CSISnapshotTimeout.Duration = c.defaultCSISnapshotTimeout
+	}
+
 	// calculate expiration
 	request.Status.Expiration = &metav1.Time{Time: c.clock.Now().Add(request.Spec.TTL.Duration)}
 
@@ -638,7 +646,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
 				backupLog.Error(err)
 			}
 
-			err = c.checkVolumeSnapshotReadyToUse(context.Background(), volumeSnapshots)
+			err = c.checkVolumeSnapshotReadyToUse(context.Background(), volumeSnapshots, backup.Spec.CSISnapshotTimeout.Duration)
 			if err != nil {
 				backupLog.Errorf("fail to wait VolumeSnapshot change to Ready: %s", err.Error())
 			}
@@ -879,9 +887,10 @@ func encodeToJSONGzip(data interface{}, desc string) (*bytes.Buffer, []error) {
 // using goroutine here instead of waiting in CSI plugin, because it's not easy to make BackupItemAction
 // parallel by now. After BackupItemAction parallel is implemented, this logic should be moved to CSI plugin
 // as https://github.com/vmware-tanzu/velero-plugin-for-csi/pull/100
-func (c *backupController) checkVolumeSnapshotReadyToUse(ctx context.Context, volumesnapshots []*snapshotv1api.VolumeSnapshot) error {
+func (c *backupController) checkVolumeSnapshotReadyToUse(ctx context.Context, volumesnapshots []*snapshotv1api.VolumeSnapshot,
+	csiSnapshotTimeout time.Duration) error {
 	eg, _ := errgroup.WithContext(ctx)
-	timeout := 10 * time.Minute
+	timeout := csiSnapshotTimeout
 	interval := 5 * time.Second
 
 	for _, vs := range volumesnapshots {

pkg/controller/backup_storage_location_controller.go

@@ -24,12 +24,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	"github.com/vmware-tanzu/velero/internal/storage"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -39,7 +37,10 @@ import (
 )
 
 const (
-	backupStorageLocationSyncPeriod = 1 * time.Minute
+	// keep the enqueue period a smaller value to make sure the BSL can be validated as expected.
+	// The BSL validation frequency is 1 minute by default, if we set the enqueue period as 1 minute,
+	// this will cause the actual validation interval for each BSL to be 2 minutes
+	bslValidationEnqueuePeriod = 10 * time.Second
 )
 
 // BackupStorageLocationReconciler reconciles a BackupStorageLocation object
@@ -185,7 +186,7 @@ func (r *BackupStorageLocationReconciler) SetupWithManager(mgr ctrl.Manager) err
 		r.Log,
 		mgr.GetClient(),
 		&velerov1api.BackupStorageLocationList{},
-		backupStorageLocationSyncPeriod,
+		bslValidationEnqueuePeriod,
 		// Add filter function to enqueue BSL per ValidationFrequency setting.
 		func(object client.Object) bool {
 			location := object.(*velerov1api.BackupStorageLocation)
@@ -193,22 +194,8 @@ func (r *BackupStorageLocationReconciler) SetupWithManager(mgr ctrl.Manager) err
 		},
 	)
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&velerov1api.BackupStorageLocation{}).
-		// Handle BSL's creation event and spec update event to let changed BSL got validation immediately.
-		WithEventFilter(predicate.Funcs{
-			CreateFunc: func(ce event.CreateEvent) bool {
-				return true
-			},
-			UpdateFunc: func(ue event.UpdateEvent) bool {
-				return ue.ObjectNew.GetGeneration() != ue.ObjectOld.GetGeneration()
-			},
-			DeleteFunc: func(de event.DeleteEvent) bool {
-				return false
-			},
-			GenericFunc: func(ge event.GenericEvent) bool {
-				return false
-			},
-		}).
+		// As the "status.LastValidationTime" field is always updated, this triggers new reconciling process, skip the update event that include no spec change to avoid the reconcile loop
+		For(&velerov1api.BackupStorageLocation{}, builder.WithPredicates(kube.SpecChangePredicate{})).
 		Watches(g, nil).
 		Complete(r)
 }

pkg/controller/pod_volume_backup_controller.go

@@ -124,7 +124,11 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	if err != nil {
 		return r.updateStatusToFailed(ctx, &pvb, err, "building Restic command", log)
 	}
-	defer os.Remove(resticDetails.credsFile)
+
+	defer func() {
+		os.Remove(resticDetails.credsFile)
+		os.Remove(resticDetails.caCertFile)
+	}()
 
 	backupLocation := &velerov1api.BackupStorageLocation{}
 	if err := r.Client.Get(context.Background(), client.ObjectKey{
@@ -204,19 +208,6 @@ func (r *PodVolumeBackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func (r *PodVolumeBackupReconciler) singlePathMatch(path string) (string, error) {
-	matches, err := r.FileSystem.Glob(path)
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	if len(matches) != 1 {
-		return "", errors.Errorf("expected one matching path: %s, got %d", path, len(matches))
-	}
-
-	return matches[0], nil
-}
-
 // getParentSnapshot finds the most recent completed PodVolumeBackup for the
 // specified PVC and returns its Restic snapshot ID. Any errors encountered are
 // logged but not returned since they do not prevent a backup from proceeding.
@@ -237,7 +228,7 @@ func (r *PodVolumeBackupReconciler) getParentSnapshot(ctx context.Context, log l
 
 	// Go through all the podvolumebackups for the PVC and look for the most
 	// recent completed one to use as the parent.
-	var mostRecentPVB *velerov1api.PodVolumeBackup
+	var mostRecentPVB velerov1api.PodVolumeBackup
 	for _, pvb := range pvbList.Items {
 		if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted {
 			continue
@@ -254,12 +245,12 @@ func (r *PodVolumeBackupReconciler) getParentSnapshot(ctx context.Context, log l
 			continue
 		}
 
-		if mostRecentPVB == nil || pvb.Status.StartTimestamp.After(mostRecentPVB.Status.StartTimestamp.Time) {
-			mostRecentPVB = &pvb
+		if mostRecentPVB.Status == (velerov1api.PodVolumeBackupStatus{}) || pvb.Status.StartTimestamp.After(mostRecentPVB.Status.StartTimestamp.Time) {
+			mostRecentPVB = pvb
 		}
 	}
 
-	if mostRecentPVB == nil {
+	if mostRecentPVB.Status == (velerov1api.PodVolumeBackupStatus{}) {
 		log.Info("No completed PodVolumeBackup found for PVC")
 		return ""
 	}
@@ -313,7 +304,7 @@ func (r *PodVolumeBackupReconciler) buildResticCommand(ctx context.Context, log
 	pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(pvb.Spec.Pod.UID), volDir)
 	log.WithField("pathGlob", pathGlob).Debug("Looking for path matching glob")
 
-	path, err := r.singlePathMatch(pathGlob)
+	path, err := kube.SinglePathMatch(pathGlob, r.FileSystem, log)
 	if err != nil {
 		return nil, errors.Wrap(err, "identifying unique volume path on host")
 	}
@@ -344,8 +335,6 @@ func (r *PodVolumeBackupReconciler) buildResticCommand(ctx context.Context, log
 		if err != nil {
 			log.WithError(err).Error("creating temporary caCert file")
 		}
-		defer os.Remove(details.caCertFile)
-
 	}
 	cmd.CACertFile = details.caCertFile
 

pkg/controller/pod_volume_restore_controller.go

@@ -215,19 +215,6 @@ func getResticInitContainerIndex(pod *corev1api.Pod) int {
 	return -1
 }
 
-func singlePathMatch(path string) (string, error) {
-	matches, err := filepath.Glob(path)
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	if len(matches) != 1 {
-		return "", errors.Errorf("expected one matching path: %s, got %d", path, len(matches))
-	}
-
-	return matches[0], nil
-}
-
 func (c *PodVolumeRestoreReconciler) processRestore(ctx context.Context, req *velerov1api.PodVolumeRestore, pod *corev1api.Pod, log logrus.FieldLogger) error {
 	volumeDir, err := kube.GetVolumeDirectory(ctx, log, pod, req.Spec.Volume, c.Client)
 	if err != nil {
@@ -236,7 +223,9 @@ func (c *PodVolumeRestoreReconciler) processRestore(ctx context.Context, req *ve
 
 	// Get the full path of the new volume's directory as mounted in the daemonset pod, which
 	// will look like: /host_pods/<new-pod-uid>/volumes/<volume-plugin-name>/<volume-dir>
-	volumePath, err := singlePathMatch(fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(req.Spec.Pod.UID), volumeDir))
+	volumePath, err := kube.SinglePathMatch(
+		fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(req.Spec.Pod.UID), volumeDir),
+		c.fileSystem, log)
 	if err != nil {
 		return errors.Wrap(err, "error identifying path of volume")
 	}

pkg/persistence/object_store.go

@@ -131,19 +131,25 @@ func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocatio
 		return nil, errors.Errorf("backup storage location's bucket name %q must not contain a '/' (if using a prefix, put it in the 'Prefix' field instead)", location.Spec.ObjectStorage.Bucket)
 	}
 
+	// Pass a new map into the object store rather than modifying the passed-in
+	// location. This prevents Velero controllers from accidentally modifying
+	// the in-cluster BSL with data which doesn't belong in Spec.Config
+	objectStoreConfig := make(map[string]string)
+	if location.Spec.Config != nil {
+		for key, val := range location.Spec.Config {
+			objectStoreConfig[key] = val
+		}
+	}
+
 	// add the bucket name and prefix to the config map so that object stores
 	// can use them when initializing. The AWS object store uses the bucket
 	// name to determine the bucket's region when setting up its client.
-	if location.Spec.Config == nil {
-		location.Spec.Config = make(map[string]string)
-	}
-
-	location.Spec.Config["bucket"] = bucket
-	location.Spec.Config["prefix"] = prefix
+	objectStoreConfig["bucket"] = bucket
+	objectStoreConfig["prefix"] = prefix
 
 	// Only include a CACert if it's specified in order to maintain compatibility with plugins that don't expect it.
 	if location.Spec.ObjectStorage.CACert != nil {
-		location.Spec.Config["caCert"] = string(location.Spec.ObjectStorage.CACert)
+		objectStoreConfig["caCert"] = string(location.Spec.ObjectStorage.CACert)
 	}
 
 	// If the BSL specifies a credential, fetch its path on disk and pass to
@@ -154,7 +160,7 @@ func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocatio
 			return nil, errors.Wrap(err, "unable to get credentials")
 		}
 
-		location.Spec.Config["credentialsFile"] = credsFile
+		objectStoreConfig["credentialsFile"] = credsFile
 	}
 
 	objectStore, err := objectStoreGetter.GetObjectStore(location.Spec.Provider)
@@ -162,7 +168,7 @@ func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocatio
 		return nil, err
 	}
 
-	if err := objectStore.Init(location.Spec.Config); err != nil {
+	if err := objectStore.Init(objectStoreConfig); err != nil {
 		return nil, err
 	}
 

pkg/plugin/framework/server.go

@@ -25,7 +25,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 
-	veleroflag "github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
 	"github.com/vmware-tanzu/velero/pkg/util/logging"
 )
 
@@ -78,6 +77,7 @@ type Server interface {
 
 	// RegisterItemSnapshotters registers multiple Item Snapshotters
 	RegisterItemSnapshotters(map[string]HandlerInitializer) Server
+
 	// Server runs the plugin server.
 	Serve()
 }
@@ -87,7 +87,6 @@ type server struct {
 	log               *logrus.Logger
 	logLevelFlag      *logging.LevelFlag
 	flagSet           *pflag.FlagSet
-	featureSet        *veleroflag.StringArray
 	backupItemAction  *BackupItemActionPlugin
 	volumeSnapshotter *VolumeSnapshotterPlugin
 	objectStore       *ObjectStorePlugin
@@ -99,12 +98,10 @@ type server struct {
 // NewServer returns a new Server
 func NewServer() Server {
 	log := newLogger()
-	features := veleroflag.NewStringArray()
 
 	return &server{
 		log:               log,
 		logLevelFlag:      logging.LogLevelFlag(log.Level),
-		featureSet:        &features,
 		backupItemAction:  NewBackupItemActionPlugin(serverLogger(log)),
 		volumeSnapshotter: NewVolumeSnapshotterPlugin(serverLogger(log)),
 		objectStore:       NewObjectStorePlugin(serverLogger(log)),
@@ -116,7 +113,6 @@ func NewServer() Server {
 
 func (s *server) BindFlags(flags *pflag.FlagSet) Server {
 	flags.Var(s.logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(s.logLevelFlag.AllowedValues(), ", ")))
-	flags.Var(s.featureSet, "features", "List of feature flags for this plugin")
 	s.flagSet = flags
 	s.flagSet.ParseErrorsWhitelist.UnknownFlags = true
 

pkg/restore/restore.go

@@ -207,11 +207,8 @@ func (kr *kubernetesRestorer) RestoreWithResolvers(
 	)
 
 	// Get resource status includes-excludes. Defaults to excluding all resources
-	restoreStatusIncludesExcludes := collections.GetResourceIncludesExcludes(
-		kr.discoveryHelper,
-		[]string{},
-		[]string{"*"},
-	)
+	var restoreStatusIncludesExcludes *collections.IncludesExcludes
+
 	if req.Restore.Spec.RestoreStatus != nil {
 		restoreStatusIncludesExcludes = collections.GetResourceIncludesExcludes(
 			kr.discoveryHelper,
@@ -1362,13 +1359,14 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 		return warnings, errs
 	}
 
-	shouldRestoreStatus := ctx.resourceStatusIncludesExcludes.ShouldInclude(groupResource.String())
+	shouldRestoreStatus := ctx.resourceStatusIncludesExcludes != nil && ctx.resourceStatusIncludesExcludes.ShouldInclude(groupResource.String())
 	if shouldRestoreStatus && statusFieldErr != nil {
 		err := fmt.Errorf("could not get status to be restored %s: %v", kube.NamespaceAndName(obj), statusFieldErr)
 		ctx.log.Errorf(err.Error())
 		errs.Add(namespace, err)
 		return warnings, errs
 	}
+	ctx.log.Debugf("status field for %s: exists: %v, should restore: %v", groupResource, statusFieldExists, shouldRestoreStatus)
 	// if it should restore status, run a UpdateStatus
 	if statusFieldExists && shouldRestoreStatus {
 		if err := unstructured.SetNestedField(obj.Object, objStatus, "status"); err != nil {
@@ -1379,6 +1377,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 		obj.SetResourceVersion(createdObj.GetResourceVersion())
 		updated, err := resourceClient.UpdateStatus(obj, metav1.UpdateOptions{})
 		if err != nil {
+			ctx.log.Infof("status field update failed %s: %v", kube.NamespaceAndName(obj), err)
 			warnings.Add(namespace, err)
 		} else {
 			createdObj = updated

pkg/util/kube/predicate.go

@@ -0,0 +1,47 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"reflect"
+
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+// SpecChangePredicate implements a default update predicate function on Spec change
+// As Velero doesn't enable subresource in CRDs, we cannot use the object's metadata.generation field to check the spec change
+// More details about the generation field refer to https://github.com/kubernetes-sigs/controller-runtime/blob/v0.12.2/pkg/predicate/predicate.go#L156
+type SpecChangePredicate struct {
+	predicate.Funcs
+}
+
+func (SpecChangePredicate) Update(e event.UpdateEvent) bool {
+	if e.ObjectOld == nil {
+		return false
+	}
+	if e.ObjectNew == nil {
+		return false
+	}
+	oldSpec := reflect.ValueOf(e.ObjectOld).Elem().FieldByName("Spec")
+	// contains no field named "Spec", return false directly
+	if oldSpec.IsZero() {
+		return false
+	}
+	newSpec := reflect.ValueOf(e.ObjectNew).Elem().FieldByName("Spec")
+	return !reflect.DeepEqual(oldSpec.Interface(), newSpec.Interface())
+}

pkg/util/kube/predicate_test.go

@@ -0,0 +1,180 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	corev1api "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+
+	velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+)
+
+func TestSpecChangePredicate(t *testing.T) {
+	cases := []struct {
+		name    string
+		oldObj  client.Object
+		newObj  client.Object
+		changed bool
+	}{
+		{
+			name: "Contains no spec field",
+			oldObj: &velerov1.BackupStorageLocation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bsl01",
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bsl01",
+				},
+			},
+			changed: false,
+		},
+		{
+			name: "ObjectMetas are different, Specs are same",
+			oldObj: &velerov1.BackupStorageLocation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        "bsl01",
+					Annotations: map[string]string{"key1": "value1"},
+				},
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        "bsl01",
+					Annotations: map[string]string{"key2": "value2"},
+				},
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+			},
+			changed: false,
+		},
+		{
+			name: "Statuses are different, Specs are same",
+			oldObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+				Status: velerov1.BackupStorageLocationStatus{
+					Phase: velerov1.BackupStorageLocationPhaseAvailable,
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+				Status: velerov1.BackupStorageLocationStatus{
+					Phase: velerov1.BackupStorageLocationPhaseUnavailable,
+				},
+			},
+			changed: false,
+		},
+		{
+			name: "Specs are different",
+			oldObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "aws",
+				},
+			},
+			changed: true,
+		},
+		{
+			name: "Specs are same",
+			oldObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+					Config:   map[string]string{"key": "value"},
+					Credential: &corev1api.SecretKeySelector{
+						LocalObjectReference: corev1api.LocalObjectReference{
+							Name: "secret",
+						},
+						Key: "credential",
+					},
+					StorageType: velerov1.StorageType{
+						ObjectStorage: &velerov1.ObjectStorageLocation{
+							Bucket: "bucket1",
+							Prefix: "prefix",
+							CACert: []byte{'a'},
+						},
+					},
+					Default:    true,
+					AccessMode: velerov1.BackupStorageLocationAccessModeReadWrite,
+					BackupSyncPeriod: &metav1.Duration{
+						Duration: 1 * time.Minute,
+					},
+					ValidationFrequency: &metav1.Duration{
+						Duration: 1 * time.Minute,
+					},
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+					Config:   map[string]string{"key": "value"},
+					Credential: &corev1api.SecretKeySelector{
+						LocalObjectReference: corev1api.LocalObjectReference{
+							Name: "secret",
+						},
+						Key: "credential",
+					},
+					StorageType: velerov1.StorageType{
+						ObjectStorage: &velerov1.ObjectStorageLocation{
+							Bucket: "bucket1",
+							Prefix: "prefix",
+							CACert: []byte{'a'},
+						},
+					},
+					Default:    true,
+					AccessMode: velerov1.BackupStorageLocationAccessModeReadWrite,
+					BackupSyncPeriod: &metav1.Duration{
+						Duration: 1 * time.Minute,
+					},
+					ValidationFrequency: &metav1.Duration{
+						Duration: 1 * time.Minute,
+					},
+				},
+			},
+			changed: false,
+		},
+	}
+
+	predicate := SpecChangePredicate{}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			changed := predicate.Update(event.UpdateEvent{
+				ObjectOld: c.oldObj,
+				ObjectNew: c.newObj,
+			})
+			assert.Equal(t, c.changed, changed)
+		})
+	}
+}

pkg/util/kube/utils.go

@@ -34,15 +34,20 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
 )
 
 // These annotations are taken from the Kubernetes persistent volume/persistent volume claim controller.
 // They cannot be directly importing because they are part of the kubernetes/kubernetes package, and importing that package is unsupported.
 // Their values are well-known and slow changing. They're duplicated here as constants to provide compile-time checking.
 // Originals can be found in kubernetes/kubernetes/pkg/controller/volume/persistentvolume/util/util.go.
-const KubeAnnBindCompleted = "pv.kubernetes.io/bind-completed"
-const KubeAnnBoundByController = "pv.kubernetes.io/bound-by-controller"
-const KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by"
+const (
+	KubeAnnBindCompleted          = "pv.kubernetes.io/bind-completed"
+	KubeAnnBoundByController      = "pv.kubernetes.io/bound-by-controller"
+	KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by"
+	KubeAnnMigratedTo             = "pv.kubernetes.io/migrated-to"
+)
 
 // NamespaceAndName returns a string in the format <namespace>/<name>
 func NamespaceAndName(objMeta metav1.Object) string {
@@ -163,6 +168,9 @@ func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1
 	return pvc.Spec.VolumeName, nil
 }
 
+// isProvisionedByCSI function checks whether this is a CSI PV by annotation.
+// Either "pv.kubernetes.io/provisioned-by" or "pv.kubernetes.io/migrated-to" indicates
+// PV is provisioned by CSI.
 func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume, kbClient client.Client) (bool, error) {
 	if pv.Spec.CSI != nil {
 		return true, nil
@@ -171,14 +179,15 @@ func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume,
 	// Refer to https://github.com/vmware-tanzu/velero/issues/4496 for more details
 	if pv.Annotations != nil {
 		driverName := pv.Annotations[KubeAnnDynamicallyProvisioned]
-		if len(driverName) > 0 {
+		migratedDriver := pv.Annotations[KubeAnnMigratedTo]
+		if len(driverName) > 0 || len(migratedDriver) > 0 {
 			list := &storagev1api.CSIDriverList{}
 			if err := kbClient.List(context.TODO(), list); err != nil {
 				return false, err
 			}
 			for _, driver := range list.Items {
-				if driverName == driver.Name {
-					log.Debugf("the annotation %s=%s indicates the volume is provisioned by a CSI driver", KubeAnnDynamicallyProvisioned, driverName)
+				if driverName == driver.Name || migratedDriver == driver.Name {
+					log.Debugf("the annotation %s or %s equals to %s indicates the volume is provisioned by a CSI driver", KubeAnnDynamicallyProvisioned, KubeAnnMigratedTo, driverName)
 					return true, nil
 				}
 			}
@@ -187,6 +196,21 @@ func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume,
 	return false, nil
 }
 
+// SinglePathMatch function will be called by PVB and PVR controller to check whether pass-in volume path is valid.
+// Check whether there is only one match by the path's pattern (/host_pods/%s/volumes/*/volume_name/[mount|]).
+func SinglePathMatch(path string, fs filesystem.Interface, log logrus.FieldLogger) (string, error) {
+	matches, err := fs.Glob(path)
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+	if len(matches) != 1 {
+		return "", errors.Errorf("expected one matching path: %s, got %d", path, len(matches))
+	}
+
+	log.Debugf("This is a valid volume path: %s.", matches[0])
+	return matches[0], nil
+}
+
 // IsV1CRDReady checks a v1 CRD to see if it's ready, with both the Established and NamesAccepted conditions.
 func IsV1CRDReady(crd *apiextv1.CustomResourceDefinition) bool {
 	var isEstablished, namesAccepted bool

pkg/util/kube/utils_test.go

@@ -197,6 +197,13 @@ func TestGetVolumeDirectorySuccess(t *testing.T) {
 			pv:   builder.ForPersistentVolume("a-pv").ObjectMeta(builder.WithAnnotations(KubeAnnDynamicallyProvisioned, "csi.test.com")).Result(),
 			want: "a-pv/mount",
 		},
+		{
+			name: "Volume with CSI annotation 'pv.kubernetes.io/migrated-to' appends '/mount' to the volume name",
+			pod:  builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(),
+			pvc:  builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(),
+			pv:   builder.ForPersistentVolume("a-pv").ObjectMeta(builder.WithAnnotations(KubeAnnMigratedTo, "csi.test.com")).Result(),
+			want: "a-pv/mount",
+		},
 	}
 
 	csiDriver := storagev1api.CSIDriver{
@@ -425,3 +432,13 @@ func TestIsCRDReady(t *testing.T) {
 	_, err = IsCRDReady(obj)
 	assert.NotNil(t, err)
 }
+
+func TestSinglePathMatch(t *testing.T) {
+	fakeFS := velerotest.NewFakeFileSystem()
+	fakeFS.MkdirAll("testDir1/subpath", 0755)
+	fakeFS.MkdirAll("testDir2/subpath", 0755)
+
+	_, err := SinglePathMatch("./*/subpath", fakeFS, logrus.StandardLogger())
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), "expected one matching path")
+}

site/content/docs/main/api-types/backup.md

@@ -29,6 +29,10 @@ metadata:
   namespace: velero
 # Parameters about the backup. Required.
 spec:
+  # CSISnapshotTimeout specifies the time used to wait for
+  # CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+  # returning error as timeout. The default value is 10 minute.
+  csiSnapshotTimeout: 10m
   # Array of namespaces to include in the backup. If unspecified, all namespaces are included.
   # Optional.
   includedNamespaces:

site/content/docs/main/api-types/schedule.md

@@ -34,6 +34,10 @@ spec:
   schedule: 0 7 * * *
   # Template is the spec that should be used for each backup triggered by this schedule.
   template:
+    # CSISnapshotTimeout specifies the time used to wait for
+    # CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+    # returning error as timeout. The default value is 10 minute.
+    csiSnapshotTimeout: 10m
     # Array of namespaces to include in the scheduled backup. If unspecified, all namespaces are included.
     # Optional.
     includedNamespaces:

site/content/docs/main/examples.md

@@ -47,10 +47,10 @@ cd velero
     kubectl apply -f examples/nginx-app/with-pv.yaml
     ```
 
-1. Create a backup with PV snapshotting:
+1. Create a backup with PV snapshotting. `--csi-snapshot-timeout` is used to setup time to wait before CSI snapshot creation timeout. The default value is 10 minutes:
 
     ```bash
-    velero backup create nginx-backup --include-namespaces nginx-example
+    velero backup create nginx-backup --include-namespaces nginx-example --csi-snapshot-timeout=20m
     ```
 
 1. Simulate a disaster:

site/content/docs/v1.9/api-types/backup.md

@@ -29,6 +29,11 @@ metadata:
   namespace: velero
 # Parameters about the backup. Required.
 spec:
+  # Available since v1.9.1.
+  # CSISnapshotTimeout specifies the time used to wait for
+  # CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+  # returning error as timeout. The default value is 10 minute.
+  csiSnapshotTimeout: 10m
   # Array of namespaces to include in the backup. If unspecified, all namespaces are included.
   # Optional.
   includedNamespaces:

site/content/docs/v1.9/api-types/schedule.md

@@ -34,6 +34,11 @@ spec:
   schedule: 0 7 * * *
   # Template is the spec that should be used for each backup triggered by this schedule.
   template:
+    # Available since v1.9.1.
+    # CSISnapshotTimeout specifies the time used to wait for
+    # CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+    # returning error as timeout. The default value is 10 minute.
+    csiSnapshotTimeout: 10m
     # Array of namespaces to include in the scheduled backup. If unspecified, all namespaces are included.
     # Optional.
     includedNamespaces:

test/e2e/bsl-mgmt/deletion.go

@@ -221,7 +221,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 					snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
 					Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
 					Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
-						VeleroCfg.CloudCredentialsFile, VeleroCfg.AdditionalBSLBucket,
+						VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
 						VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
 				})
 				By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
@@ -232,6 +232,8 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 						BSLCredentials = VeleroCfg.AdditionalBSLCredentials
 						BSLConfig = VeleroCfg.AdditionalBSLConfig
 					} else {
+						// Snapshotting by non-vSphere provider using credentials
+						//    and config in default BSL
 						BSLCredentials = VeleroCfg.CloudCredentialsFile
 						BSLConfig = VeleroCfg.BSLConfig
 					}

test/e2e/util/providers/common.go

@@ -36,9 +36,9 @@ type ObjectsInStorage interface {
 
 func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
 	fmt.Printf("|| VERIFICATION || - %s %s should exist in storage [%s]\n", subPrefix, backupName, bslPrefix)
-	exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
+	exist, err := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 	if !exist {
-		return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected\n", backupName))
+		return errors.Wrap(err, fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected\n", backupName))
 	}
 	fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket)
 	return nil

test/e2e/util/velero/velero_utils.go

@@ -87,6 +87,13 @@ var pluginsMatrix = map[string]map[string][]string{
 		"gcp":       {"velero/velero-plugin-for-gcp:v1.4.0"},
 		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.4.0", "velero/velero-plugin-for-csi:v0.2.0"},
 	},
+	"v1.9": {
+		"aws":       {"velero/velero-plugin-for-aws:v1.5.0"},
+		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.5.0"},
+		"vsphere":   {"velero/velero-plugin-for-aws:v1.5.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"},
+		"gcp":       {"velero/velero-plugin-for-gcp:v1.5.0"},
+		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.5.0", "velero/velero-plugin-for-csi:v0.3.0"},
+	},
 	"main": {
 		"aws":       {"velero/velero-plugin-for-aws:main"},
 		"azure":     {"velero/velero-plugin-for-microsoft-azure:main"},