Generate the changelog for v1.9.1 (#5176)

Signed-off-by: Ming <mqiu@vmware.com>

.goreleaser.yml

@@ -14,7 +14,7 @@
 
 dist: _output
 builds:
-  - main: ./cmd/velero/main.go
+  - main: ./cmd/velero/velero.go
     env:
       - CGO_ENABLED=0
     goos:

changelogs/CHANGELOG-1.9.md

@@ -1,3 +1,26 @@
+## v1.9.1
+### 2022-08-03
+
+### Download
+https://github.com/vmware-tanzu/velero/releases/tag/v1.9.1
+
+### Container Image
+`velero/velero:v1.9.1`
+
+### Documentation
+https://velero.io/docs/v1.9/
+
+### Upgrading
+https://velero.io/docs/v1.9/upgrade-to-1.9/
+
+### All changes
+
+  * Fix bsl validation bug: the BSL is validated continually and doesn't respect the validation period configured (#5112, @ywk253100)
+  * Modify BackupStoreGetter to avoid BSL spec changes (#5134, @sseago)
+  * Delay CA file deletion in PVB controller. (#5150, @jxun)
+  * Skip registering "crd-remap-version" plugin when feature flag "EnableAPIGroupVersions" is set (#5173, @reasonerjt)
+  * Fix restic backups to multiple backup storage locations bug (#5175, @qiuming-best)
+
 ## v1.9.0
 ### 2022-06-13
 
@@ -102,3 +125,4 @@ With bumping up the API to v1 in CSI plugin, the v0.3.0 CSI plugin will only wor
   * Fix E2E test [Backups][Deletion][Restic] on GCP. (#4968, @jxun)
   * Disable status as sub resource in CRDs (#4972, @ywk253100)
   * Add more information for failing to get path or snapshot in restic backup and restore. (#4988, @jxun)
+  * When spec.RestoreStatus is empty, don't restore status (#5015, @sseago)

pkg/cmd/server/plugin/plugin.go

@@ -19,9 +19,11 @@ package plugin
 import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/features"
+
 	"github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/client"
 	velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery"
@@ -36,11 +38,10 @@ func NewCommand(f client.Factory) *cobra.Command {
 		Hidden: true,
 		Short:  "INTERNAL COMMAND ONLY - not intended to be run directly by users",
 		Run: func(c *cobra.Command, args []string) {
-			pluginServer.
+			pluginServer = pluginServer.
 				RegisterBackupItemAction("velero.io/pv", newPVBackupItemAction).
 				RegisterBackupItemAction("velero.io/pod", newPodBackupItemAction).
 				RegisterBackupItemAction("velero.io/service-account", newServiceAccountBackupItemAction(f)).
-				RegisterBackupItemAction("velero.io/crd-remap-version", newRemapCRDVersionAction(f)).
 				RegisterRestoreItemAction("velero.io/job", newJobRestoreItemAction).
 				RegisterRestoreItemAction("velero.io/pod", newPodRestoreItemAction).
 				RegisterRestoreItemAction("velero.io/restic", newResticRestoreItemAction(f)).
@@ -55,13 +56,15 @@ func NewCommand(f client.Factory) *cobra.Command {
 				RegisterRestoreItemAction("velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction).
 				RegisterRestoreItemAction("velero.io/change-pvc-node-selector", newChangePVCNodeSelectorItemAction(f)).
 				RegisterRestoreItemAction("velero.io/apiservice", newAPIServiceRestoreItemAction).
-				RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction).
-				Serve()
+				RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction)
+			if !features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag) {
+				// Do not register crd-remap-version BIA if the API Group feature flag is enabled, so that the v1 CRD can be backed up
+				pluginServer = pluginServer.RegisterBackupItemAction("velero.io/crd-remap-version", newRemapCRDVersionAction(f))
+			}
+			pluginServer.Serve()
 		},
 	}
-
 	pluginServer.BindFlags(c.Flags())
-
 	return c
 }
 

pkg/controller/backup_storage_location_controller.go

@@ -24,12 +24,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	"github.com/vmware-tanzu/velero/internal/storage"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -39,7 +37,10 @@ import (
 )
 
 const (
-	backupStorageLocationSyncPeriod = 1 * time.Minute
+	// keep the enqueue period a smaller value to make sure the BSL can be validated as expected.
+	// The BSL validation frequency is 1 minute by default, if we set the enqueue period as 1 minute,
+	// this will cause the actual validation interval for each BSL to be 2 minutes
+	bslValidationEnqueuePeriod = 10 * time.Second
 )
 
 // BackupStorageLocationReconciler reconciles a BackupStorageLocation object
@@ -185,7 +186,7 @@ func (r *BackupStorageLocationReconciler) SetupWithManager(mgr ctrl.Manager) err
 		r.Log,
 		mgr.GetClient(),
 		&velerov1api.BackupStorageLocationList{},
-		backupStorageLocationSyncPeriod,
+		bslValidationEnqueuePeriod,
 		// Add filter function to enqueue BSL per ValidationFrequency setting.
 		func(object client.Object) bool {
 			location := object.(*velerov1api.BackupStorageLocation)
@@ -193,22 +194,8 @@ func (r *BackupStorageLocationReconciler) SetupWithManager(mgr ctrl.Manager) err
 		},
 	)
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&velerov1api.BackupStorageLocation{}).
-		// Handle BSL's creation event and spec update event to let changed BSL got validation immediately.
-		WithEventFilter(predicate.Funcs{
-			CreateFunc: func(ce event.CreateEvent) bool {
-				return true
-			},
-			UpdateFunc: func(ue event.UpdateEvent) bool {
-				return ue.ObjectNew.GetGeneration() != ue.ObjectOld.GetGeneration()
-			},
-			DeleteFunc: func(de event.DeleteEvent) bool {
-				return false
-			},
-			GenericFunc: func(ge event.GenericEvent) bool {
-				return false
-			},
-		}).
+		// As the "status.LastValidationTime" field is always updated, this triggers new reconciling process, skip the update event that include no spec change to avoid the reconcile loop
+		For(&velerov1api.BackupStorageLocation{}, builder.WithPredicates(kube.SpecChangePredicate{})).
 		Watches(g, nil).
 		Complete(r)
 }

pkg/controller/pod_volume_backup_controller.go

@@ -124,7 +124,11 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	if err != nil {
 		return r.updateStatusToFailed(ctx, &pvb, err, "building Restic command", log)
 	}
-	defer os.Remove(resticDetails.credsFile)
+
+	defer func() {
+		os.Remove(resticDetails.credsFile)
+		os.Remove(resticDetails.caCertFile)
+	}()
 
 	backupLocation := &velerov1api.BackupStorageLocation{}
 	if err := r.Client.Get(context.Background(), client.ObjectKey{
@@ -237,7 +241,7 @@ func (r *PodVolumeBackupReconciler) getParentSnapshot(ctx context.Context, log l
 
 	// Go through all the podvolumebackups for the PVC and look for the most
 	// recent completed one to use as the parent.
-	var mostRecentPVB *velerov1api.PodVolumeBackup
+	var mostRecentPVB velerov1api.PodVolumeBackup
 	for _, pvb := range pvbList.Items {
 		if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted {
 			continue
@@ -254,12 +258,12 @@ func (r *PodVolumeBackupReconciler) getParentSnapshot(ctx context.Context, log l
 			continue
 		}
 
-		if mostRecentPVB == nil || pvb.Status.StartTimestamp.After(mostRecentPVB.Status.StartTimestamp.Time) {
-			mostRecentPVB = &pvb
+		if mostRecentPVB.Status == (velerov1api.PodVolumeBackupStatus{}) || pvb.Status.StartTimestamp.After(mostRecentPVB.Status.StartTimestamp.Time) {
+			mostRecentPVB = pvb
 		}
 	}
 
-	if mostRecentPVB == nil {
+	if mostRecentPVB.Status == (velerov1api.PodVolumeBackupStatus{}) {
 		log.Info("No completed PodVolumeBackup found for PVC")
 		return ""
 	}
@@ -344,8 +348,6 @@ func (r *PodVolumeBackupReconciler) buildResticCommand(ctx context.Context, log
 		if err != nil {
 			log.WithError(err).Error("creating temporary caCert file")
 		}
-		defer os.Remove(details.caCertFile)
-
 	}
 	cmd.CACertFile = details.caCertFile
 

pkg/persistence/object_store.go

@@ -131,19 +131,25 @@ func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocatio
 		return nil, errors.Errorf("backup storage location's bucket name %q must not contain a '/' (if using a prefix, put it in the 'Prefix' field instead)", location.Spec.ObjectStorage.Bucket)
 	}
 
+	// Pass a new map into the object store rather than modifying the passed-in
+	// location. This prevents Velero controllers from accidentally modifying
+	// the in-cluster BSL with data which doesn't belong in Spec.Config
+	objectStoreConfig := make(map[string]string)
+	if location.Spec.Config != nil {
+		for key, val := range location.Spec.Config {
+			objectStoreConfig[key] = val
+		}
+	}
+
 	// add the bucket name and prefix to the config map so that object stores
 	// can use them when initializing. The AWS object store uses the bucket
 	// name to determine the bucket's region when setting up its client.
-	if location.Spec.Config == nil {
-		location.Spec.Config = make(map[string]string)
-	}
-
-	location.Spec.Config["bucket"] = bucket
-	location.Spec.Config["prefix"] = prefix
+	objectStoreConfig["bucket"] = bucket
+	objectStoreConfig["prefix"] = prefix
 
 	// Only include a CACert if it's specified in order to maintain compatibility with plugins that don't expect it.
 	if location.Spec.ObjectStorage.CACert != nil {
-		location.Spec.Config["caCert"] = string(location.Spec.ObjectStorage.CACert)
+		objectStoreConfig["caCert"] = string(location.Spec.ObjectStorage.CACert)
 	}
 
 	// If the BSL specifies a credential, fetch its path on disk and pass to
@@ -154,7 +160,7 @@ func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocatio
 			return nil, errors.Wrap(err, "unable to get credentials")
 		}
 
-		location.Spec.Config["credentialsFile"] = credsFile
+		objectStoreConfig["credentialsFile"] = credsFile
 	}
 
 	objectStore, err := objectStoreGetter.GetObjectStore(location.Spec.Provider)
@@ -162,7 +168,7 @@ func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocatio
 		return nil, err
 	}
 
-	if err := objectStore.Init(location.Spec.Config); err != nil {
+	if err := objectStore.Init(objectStoreConfig); err != nil {
 		return nil, err
 	}
 

pkg/plugin/framework/server.go

@@ -25,7 +25,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 
-	veleroflag "github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
 	"github.com/vmware-tanzu/velero/pkg/util/logging"
 )
 
@@ -78,6 +77,7 @@ type Server interface {
 
 	// RegisterItemSnapshotters registers multiple Item Snapshotters
 	RegisterItemSnapshotters(map[string]HandlerInitializer) Server
+
 	// Server runs the plugin server.
 	Serve()
 }
@@ -87,7 +87,6 @@ type server struct {
 	log               *logrus.Logger
 	logLevelFlag      *logging.LevelFlag
 	flagSet           *pflag.FlagSet
-	featureSet        *veleroflag.StringArray
 	backupItemAction  *BackupItemActionPlugin
 	volumeSnapshotter *VolumeSnapshotterPlugin
 	objectStore       *ObjectStorePlugin
@@ -99,12 +98,10 @@ type server struct {
 // NewServer returns a new Server
 func NewServer() Server {
 	log := newLogger()
-	features := veleroflag.NewStringArray()
 
 	return &server{
 		log:               log,
 		logLevelFlag:      logging.LogLevelFlag(log.Level),
-		featureSet:        &features,
 		backupItemAction:  NewBackupItemActionPlugin(serverLogger(log)),
 		volumeSnapshotter: NewVolumeSnapshotterPlugin(serverLogger(log)),
 		objectStore:       NewObjectStorePlugin(serverLogger(log)),
@@ -116,7 +113,6 @@ func NewServer() Server {
 
 func (s *server) BindFlags(flags *pflag.FlagSet) Server {
 	flags.Var(s.logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(s.logLevelFlag.AllowedValues(), ", ")))
-	flags.Var(s.featureSet, "features", "List of feature flags for this plugin")
 	s.flagSet = flags
 	s.flagSet.ParseErrorsWhitelist.UnknownFlags = true
 

pkg/restore/restore.go

@@ -207,11 +207,8 @@ func (kr *kubernetesRestorer) RestoreWithResolvers(
 	)
 
 	// Get resource status includes-excludes. Defaults to excluding all resources
-	restoreStatusIncludesExcludes := collections.GetResourceIncludesExcludes(
-		kr.discoveryHelper,
-		[]string{},
-		[]string{"*"},
-	)
+	var restoreStatusIncludesExcludes *collections.IncludesExcludes
+
 	if req.Restore.Spec.RestoreStatus != nil {
 		restoreStatusIncludesExcludes = collections.GetResourceIncludesExcludes(
 			kr.discoveryHelper,
@@ -1362,13 +1359,14 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 		return warnings, errs
 	}
 
-	shouldRestoreStatus := ctx.resourceStatusIncludesExcludes.ShouldInclude(groupResource.String())
+	shouldRestoreStatus := ctx.resourceStatusIncludesExcludes != nil && ctx.resourceStatusIncludesExcludes.ShouldInclude(groupResource.String())
 	if shouldRestoreStatus && statusFieldErr != nil {
 		err := fmt.Errorf("could not get status to be restored %s: %v", kube.NamespaceAndName(obj), statusFieldErr)
 		ctx.log.Errorf(err.Error())
 		errs.Add(namespace, err)
 		return warnings, errs
 	}
+	ctx.log.Debugf("status field for %s: exists: %v, should restore: %v", groupResource, statusFieldExists, shouldRestoreStatus)
 	// if it should restore status, run a UpdateStatus
 	if statusFieldExists && shouldRestoreStatus {
 		if err := unstructured.SetNestedField(obj.Object, objStatus, "status"); err != nil {
@@ -1379,6 +1377,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 		obj.SetResourceVersion(createdObj.GetResourceVersion())
 		updated, err := resourceClient.UpdateStatus(obj, metav1.UpdateOptions{})
 		if err != nil {
+			ctx.log.Infof("status field update failed %s: %v", kube.NamespaceAndName(obj), err)
 			warnings.Add(namespace, err)
 		} else {
 			createdObj = updated

pkg/util/kube/predicate.go

@@ -0,0 +1,47 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"reflect"
+
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+// SpecChangePredicate implements a default update predicate function on Spec change
+// As Velero doesn't enable subresource in CRDs, we cannot use the object's metadata.generation field to check the spec change
+// More details about the generation field refer to https://github.com/kubernetes-sigs/controller-runtime/blob/v0.12.2/pkg/predicate/predicate.go#L156
+type SpecChangePredicate struct {
+	predicate.Funcs
+}
+
+func (SpecChangePredicate) Update(e event.UpdateEvent) bool {
+	if e.ObjectOld == nil {
+		return false
+	}
+	if e.ObjectNew == nil {
+		return false
+	}
+	oldSpec := reflect.ValueOf(e.ObjectOld).Elem().FieldByName("Spec")
+	// contains no field named "Spec", return false directly
+	if oldSpec.IsZero() {
+		return false
+	}
+	newSpec := reflect.ValueOf(e.ObjectNew).Elem().FieldByName("Spec")
+	return !reflect.DeepEqual(oldSpec.Interface(), newSpec.Interface())
+}

pkg/util/kube/predicate_test.go

@@ -0,0 +1,180 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	corev1api "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+
+	velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+)
+
+func TestSpecChangePredicate(t *testing.T) {
+	cases := []struct {
+		name    string
+		oldObj  client.Object
+		newObj  client.Object
+		changed bool
+	}{
+		{
+			name: "Contains no spec field",
+			oldObj: &velerov1.BackupStorageLocation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bsl01",
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bsl01",
+				},
+			},
+			changed: false,
+		},
+		{
+			name: "ObjectMetas are different, Specs are same",
+			oldObj: &velerov1.BackupStorageLocation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        "bsl01",
+					Annotations: map[string]string{"key1": "value1"},
+				},
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        "bsl01",
+					Annotations: map[string]string{"key2": "value2"},
+				},
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+			},
+			changed: false,
+		},
+		{
+			name: "Statuses are different, Specs are same",
+			oldObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+				Status: velerov1.BackupStorageLocationStatus{
+					Phase: velerov1.BackupStorageLocationPhaseAvailable,
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+				Status: velerov1.BackupStorageLocationStatus{
+					Phase: velerov1.BackupStorageLocationPhaseUnavailable,
+				},
+			},
+			changed: false,
+		},
+		{
+			name: "Specs are different",
+			oldObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "aws",
+				},
+			},
+			changed: true,
+		},
+		{
+			name: "Specs are same",
+			oldObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+					Config:   map[string]string{"key": "value"},
+					Credential: &corev1api.SecretKeySelector{
+						LocalObjectReference: corev1api.LocalObjectReference{
+							Name: "secret",
+						},
+						Key: "credential",
+					},
+					StorageType: velerov1.StorageType{
+						ObjectStorage: &velerov1.ObjectStorageLocation{
+							Bucket: "bucket1",
+							Prefix: "prefix",
+							CACert: []byte{'a'},
+						},
+					},
+					Default:    true,
+					AccessMode: velerov1.BackupStorageLocationAccessModeReadWrite,
+					BackupSyncPeriod: &metav1.Duration{
+						Duration: 1 * time.Minute,
+					},
+					ValidationFrequency: &metav1.Duration{
+						Duration: 1 * time.Minute,
+					},
+				},
+			},
+			newObj: &velerov1.BackupStorageLocation{
+				Spec: velerov1.BackupStorageLocationSpec{
+					Provider: "azure",
+					Config:   map[string]string{"key": "value"},
+					Credential: &corev1api.SecretKeySelector{
+						LocalObjectReference: corev1api.LocalObjectReference{
+							Name: "secret",
+						},
+						Key: "credential",
+					},
+					StorageType: velerov1.StorageType{
+						ObjectStorage: &velerov1.ObjectStorageLocation{
+							Bucket: "bucket1",
+							Prefix: "prefix",
+							CACert: []byte{'a'},
+						},
+					},
+					Default:    true,
+					AccessMode: velerov1.BackupStorageLocationAccessModeReadWrite,
+					BackupSyncPeriod: &metav1.Duration{
+						Duration: 1 * time.Minute,
+					},
+					ValidationFrequency: &metav1.Duration{
+						Duration: 1 * time.Minute,
+					},
+				},
+			},
+			changed: false,
+		},
+	}
+
+	predicate := SpecChangePredicate{}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			changed := predicate.Update(event.UpdateEvent{
+				ObjectOld: c.oldObj,
+				ObjectNew: c.newObj,
+			})
+			assert.Equal(t, c.changed, changed)
+		})
+	}
+}

test/e2e/bsl-mgmt/deletion.go

@@ -221,7 +221,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 					snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
 					Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
 					Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
-						VeleroCfg.CloudCredentialsFile, VeleroCfg.AdditionalBSLBucket,
+						VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
 						VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
 				})
 				By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
@@ -232,6 +232,8 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 						BSLCredentials = VeleroCfg.AdditionalBSLCredentials
 						BSLConfig = VeleroCfg.AdditionalBSLConfig
 					} else {
+						// Snapshotting by non-vSphere provider using credentials
+						//    and config in default BSL
 						BSLCredentials = VeleroCfg.CloudCredentialsFile
 						BSLConfig = VeleroCfg.BSLConfig
 					}

test/e2e/util/providers/common.go

@@ -36,9 +36,9 @@ type ObjectsInStorage interface {
 
 func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
 	fmt.Printf("|| VERIFICATION || - %s %s should exist in storage [%s]\n", subPrefix, backupName, bslPrefix)
-	exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
+	exist, err := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 	if !exist {
-		return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected\n", backupName))
+		return errors.Wrap(err, fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected\n", backupName))
 	}
 	fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket)
 	return nil

test/e2e/util/velero/velero_utils.go

@@ -87,6 +87,13 @@ var pluginsMatrix = map[string]map[string][]string{
 		"gcp":       {"velero/velero-plugin-for-gcp:v1.4.0"},
 		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.4.0", "velero/velero-plugin-for-csi:v0.2.0"},
 	},
+	"v1.9": {
+		"aws":       {"velero/velero-plugin-for-aws:v1.5.0"},
+		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.5.0"},
+		"vsphere":   {"velero/velero-plugin-for-aws:v1.5.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"},
+		"gcp":       {"velero/velero-plugin-for-gcp:v1.5.0"},
+		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.5.0", "velero/velero-plugin-for-csi:v0.3.0"},
+	},
 	"main": {
 		"aws":       {"velero/velero-plugin-for-aws:main"},
 		"azure":     {"velero/velero-plugin-for-microsoft-azure:main"},