Merge pull request #4896 from danfengliu/add-backup-ttl-e2e-test

Add setting TTL in backup E2E test

test/e2e/backups/deletion.go

@@ -118,7 +118,13 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
 	if err != nil {
 		return err
 	}
-	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots, ""); err != nil {
+	var BackupCfg BackupConfig
+	BackupCfg.BackupName = backupName
+	BackupCfg.Namespace = deletionTest
+	BackupCfg.BackupLocation = backupLocation
+	BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
+	BackupCfg.Selector = ""
+	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil {
 		// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
 		// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
 		VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName)
@@ -141,9 +147,6 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
 	if useVolumeSnapshots {
 		snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList)
 		Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
-		if err != nil {
-			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
-		}
 		err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
 			VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig,
 			backupName, snapshotCheckPoint)
@@ -185,7 +188,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
 	}
 
 	backupName = "backup-1-" + UUIDgen.String()
-	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots, ""); err != nil {
+	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil {
 		// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
 		// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
 		VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName)

test/e2e/backups/sync_backups.go

@@ -89,8 +89,14 @@ func BackupsSyncTest() {
 			Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
 		}()
 
+		var BackupCfg BackupConfig
+		BackupCfg.BackupName = test.backupName
+		BackupCfg.Namespace = test.testNS
+		BackupCfg.BackupLocation = ""
+		BackupCfg.UseVolumeSnapshots = false
+		BackupCfg.Selector = ""
 		By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
-			if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false, ""); err != nil {
+			if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg); err != nil {
 				RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
 			}
 			Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS))
@@ -116,14 +122,21 @@ func BackupsSyncTest() {
 			Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(),
 				fmt.Sprintf("Failed to create %s namespace", test.testNS))
 		})
+		if !VeleroCfg.Debug {
+			defer func() {
+				Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(),
+					fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
+			}()
+		}
 
-		defer func() {
-			Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(),
-				fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
-		}()
-
+		var BackupCfg BackupConfig
+		BackupCfg.BackupName = test.backupName
+		BackupCfg.Namespace = test.testNS
+		BackupCfg.BackupLocation = ""
+		BackupCfg.UseVolumeSnapshots = false
+		BackupCfg.Selector = ""
 		By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
-			if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false, ""); err != nil {
+			if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg); err != nil {
 				RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
 			}
 			Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS))

test/e2e/backups/ttl.go

@@ -0,0 +1,200 @@
+/*
+ *
+ * Copyright the Velero contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package backups
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "github.com/vmware-tanzu/velero/test/e2e"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+
+	. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
+)
+
+type TTL struct {
+	testNS      string
+	backupName  string
+	restoreName string
+	ctx         context.Context
+	ttl         time.Duration
+}
+
+func (b *TTL) Init() {
+	rand.Seed(time.Now().UnixNano())
+	UUIDgen, _ = uuid.NewRandom()
+	b.testNS = "backup-ttl-test-" + UUIDgen.String()
+	b.backupName = "backup-ttl-test-" + UUIDgen.String()
+	b.restoreName = "restore-ttl-test-" + UUIDgen.String()
+	b.ctx, _ = context.WithTimeout(context.Background(), time.Duration(time.Minute*30))
+	b.ttl = time.Duration(10 * time.Minute)
+
+}
+
+func TTLTest() {
+	useVolumeSnapshots := true
+	test := new(TTL)
+	client, err := NewTestClient()
+	if err != nil {
+		println(err.Error())
+	}
+	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
+
+	BeforeEach(func() {
+		flag.Parse()
+		if VeleroCfg.InstallVelero {
+			// Make sure GCFrequency is shorter than backup TTL
+			VeleroCfg.GCFrequency = "4m0s"
+			Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
+		}
+	})
+
+	AfterEach(func() {
+		if VeleroCfg.InstallVelero {
+			VeleroCfg.GCFrequency = ""
+			if !VeleroCfg.Debug {
+				Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
+				Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
+			}
+		}
+	})
+
+	It("Backups in object storage should be synced to a new Velero successfully", func() {
+		test.Init()
+		oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
+		By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS), func() {
+			Expect(CreateNamespace(oneHourTimeout, client, test.testNS)).To(Succeed(),
+				fmt.Sprintf("Failed to create %s namespace", test.testNS))
+		})
+
+		By("Deploy sample workload of Kibishii", func() {
+			Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, VeleroCfg.CloudProvider,
+				test.testNS, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features,
+				VeleroCfg.KibishiiDirectory, useVolumeSnapshots)).To(Succeed())
+		})
+
+		var BackupCfg BackupConfig
+		BackupCfg.BackupName = test.backupName
+		BackupCfg.Namespace = test.testNS
+		BackupCfg.BackupLocation = ""
+		BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
+		BackupCfg.Selector = ""
+		BackupCfg.TTL = test.ttl
+
+		By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
+			Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
+				RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
+				return "Fail to backup workload"
+			})
+		})
+
+		var snapshotCheckPoint SnapshotCheckPoint
+		if useVolumeSnapshots {
+			if VeleroCfg.CloudProvider == "vsphere" {
+				// TODO - remove after upload progress monitoring is implemented
+				By("Waiting for vSphere uploads to complete", func() {
+					Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
+						test.testNS)).To(Succeed())
+				})
+			}
+			snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList)
+			Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+
+			Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
+				VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig,
+				test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+		}
+
+		By(fmt.Sprintf("Simulating a disaster by removing namespace %s\n", BackupCfg.BackupName), func() {
+			Expect(DeleteNamespace(test.ctx, client, BackupCfg.BackupName, true)).To(Succeed(),
+				fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName))
+		})
+
+		if VeleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
+			fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...")
+			time.Sleep(7 * time.Minute)
+		}
+
+		By(fmt.Sprintf("Restore %s", test.testNS), func() {
+			Expect(VeleroRestore(test.ctx, VeleroCfg.VeleroCLI,
+				VeleroCfg.VeleroNamespace, test.restoreName, test.backupName)).To(Succeed(), func() string {
+				RunDebug(test.ctx, VeleroCfg.VeleroCLI,
+					VeleroCfg.VeleroNamespace, "", test.restoreName)
+				return "Fail to restore workload"
+			})
+		})
+
+		By("Associated Restores should be created", func() {
+			Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider,
+				VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName,
+				RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object")
+
+		})
+
+		By("Check TTL was set correctly", func() {
+			ttl, err := GetBackupTTL(test.ctx, VeleroCfg.VeleroNamespace, test.backupName)
+			Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+			t, _ := time.ParseDuration(strings.ReplaceAll(ttl, "'", ""))
+			fmt.Println(t.Round(time.Minute).String())
+			Expect(t).To(Equal(test.ttl))
+		})
+
+		By(fmt.Sprintf("Waiting %s minutes for removing backup ralated resources by GC", test.ttl.String()), func() {
+			time.Sleep(test.ttl)
+		})
+
+		By("Check if backups are deleted by GC", func() {
+			Expect(WaitBackupDeleted(test.ctx, VeleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Backup %s was not deleted by GC", test.backupName))
+		})
+
+		By("Backup file from cloud object storage should be deleted", func() {
+			Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider,
+				VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName,
+				BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+		})
+
+		By("PersistentVolume snapshots should be deleted", func() {
+			if useVolumeSnapshots {
+				Expect(WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
+					VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig,
+					test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+			}
+		})
+
+		By("Associated Restores should be deleted", func() {
+			Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider,
+				VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName,
+				RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object")
+
+		})
+	})
+}

test/e2e/basic/enable_api_group_versions.go

@@ -242,7 +242,13 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso
 		backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
 		namespacesStr := strings.Join(tc.namespaces, ",")
 
-		err = VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false, "")
+		var BackupCfg BackupConfig
+		BackupCfg.BackupName = backup
+		BackupCfg.Namespace = namespacesStr
+		BackupCfg.BackupLocation = ""
+		BackupCfg.UseVolumeSnapshots = false
+		BackupCfg.Selector = ""
+		err = VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)
 		if err != nil {
 			RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "")
 			return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr)

test/e2e/bsl-mgmt/deletion.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"strings"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
@@ -29,7 +28,6 @@ import (
 	. "github.com/onsi/gomega"
 
 	. "github.com/vmware-tanzu/velero/test/e2e"
-	util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
 
@@ -179,18 +177,25 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 				Expect(AddLabelToPvc(context.Background(), pvc2, bslDeletionTestNs, label_2)).To(Succeed())
 			})
 
+			var BackupCfg BackupConfig
+			BackupCfg.BackupName = backupName_1
+			BackupCfg.Namespace = bslDeletionTestNs
+			BackupCfg.BackupLocation = backupLocation_1
+			BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
+			BackupCfg.Selector = label_1
 			By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation_1), func() {
 				// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
 				// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
 				Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI,
-					VeleroCfg.VeleroNamespace, backupName_1, bslDeletionTestNs,
-					backupLocation_1, useVolumeSnapshots, label_1)).To(Succeed())
+					VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed())
 			})
 
+			BackupCfg.BackupName = backupName_2
+			BackupCfg.BackupLocation = backupLocation_2
+			BackupCfg.Selector = label_2
 			By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() {
 				Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI,
-					VeleroCfg.VeleroNamespace, backupName_2, bslDeletionTestNs,
-					backupLocation_2, useVolumeSnapshots, label_2)).To(Succeed())
+					VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed())
 			})
 
 			if useVolumeSnapshots {
@@ -208,14 +213,6 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 						Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
 							backupName_2, 1)).To(Succeed())
 					})
-				} else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
-					By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_1), func() {
-						Expect(util.CheckVolumeSnapshotCR(client, []string{podName_1}, bslDeletionTestNs, backupName_1)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.")
-					})
-
-					By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_2), func() {
-						Expect(util.CheckVolumeSnapshotCR(client, []string{podName_2}, bslDeletionTestNs, backupName_2)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.")
-					})
 				}
 
 				var snapshotCheckPoint SnapshotCheckPoint

test/e2e/e2e_suite_test.go

@@ -65,6 +65,7 @@ func init() {
 	flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
 	flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.")
 	flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "Switch to control namespace cleaning.")
+	flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "Frequency of garbage collection.")
 }
 
 var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest)
@@ -93,12 +94,14 @@ var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on
 var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources)
 var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources)
 var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector)
-var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
 
+var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
 var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
-var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
+var _ = Describe("[Backups][TTL] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
 var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)
 
+var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
+
 var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots)
 var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic)
 

test/e2e/types.go

@@ -17,6 +17,8 @@ limitations under the License.
 package e2e
 
 import (
+	"time"
+
 	"github.com/google/uuid"
 )
 
@@ -51,6 +53,7 @@ type VerleroConfig struct {
 	KibishiiDirectory        string
 	Features                 string
 	Debug                    bool
+	GCFrequency              string
 }
 
 type SnapshotCheckPoint struct {
@@ -62,3 +65,12 @@ type SnapshotCheckPoint struct {
 	PodName        []string
 	EnableCSI      bool
 }
+
+type BackupConfig struct {
+	BackupName         string
+	Namespace          string
+	BackupLocation     string
+	UseVolumeSnapshots bool
+	Selector           string
+	TTL                time.Duration
+}

test/e2e/upgrade/upgrade.go

@@ -27,7 +27,6 @@ import (
 	. "github.com/onsi/gomega"
 
 	. "github.com/vmware-tanzu/velero/test/e2e"
-	util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
@@ -79,6 +78,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
 		client                  TestClient
 		err                     error
 	)
+
 	By("Create test client instance", func() {
 		client, err = NewTestClient()
 		Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
@@ -125,7 +125,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
 					Expect(err).To(Succeed())
 				})
 			}
-
+			VeleroCfg.GCFrequency = ""
 			By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade",
 				upgradeFromVelero.UpgradeFromVeleroVersion), func() {
 				//Set VeleroImage and ResticHelperImage to blank
@@ -162,9 +162,14 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
 			})
 
 			By(fmt.Sprintf("Backup namespace %s", upgradeNamespace), func() {
+				var BackupCfg BackupConfig
+				BackupCfg.BackupName = backupName
+				BackupCfg.Namespace = upgradeNamespace
+				BackupCfg.BackupLocation = ""
+				BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
+				BackupCfg.Selector = ""
 				Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI,
-					tmpCfg.VeleroNamespace, backupName, upgradeNamespace, "",
-					useVolumeSnapshots, "")).ShouldNot(HaveOccurred(), func() string {
+					tmpCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string {
 					err = VeleroBackupLogs(context.Background(), tmpCfg.UpgradeFromVeleroCLI,
 						tmpCfg.VeleroNamespace, backupName)
 					return "Get backup logs"
@@ -178,10 +183,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
 						Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
 							upgradeNamespace)).To(Succeed())
 					})
-				} else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
-					By("CSI VolumeSnapshotContent CR should be created", func() {
-						Expect(util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, upgradeNamespace, backupName)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content")
-					})
 				}
 				var snapshotCheckPoint SnapshotCheckPoint
 				snapshotCheckPoint.NamespaceBackedUp = upgradeNamespace
@@ -206,13 +207,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
 				By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
 					time.Sleep(5 * time.Minute)
 				})
-				// TODO: add WaitUntilSnapshotsNotExistInCloud verification when Upgrade test is enabled in nightly
-				// err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
-				// 	VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
-				// 	backupName, snapshotCheckPoint)
-				// if err != nil {
-				// 	return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
-				// }
 			}
 			// the snapshots of AWS may be still in pending status when do the restore, wait for a while
 			// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
@@ -223,6 +217,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
 			}
 
 			By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() {
+				tmpCfg.GCFrequency = ""
 				Expect(VeleroInstall(context.Background(), &tmpCfg, useVolumeSnapshots)).To(Succeed())
 				Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI,
 					tmpCfg.VeleroVersion)).To(Succeed())

test/e2e/util/csi/common.go

@@ -65,11 +65,20 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error
 	}
 	var snapshotHandleList []string
 	for _, i := range vscList.Items {
+		if i.Status == nil {
+			fmt.Println("SnapshotHandle Status s nil")
+			continue
+		}
 		if i.Status.SnapshotHandle == nil {
 			fmt.Println("SnapshotHandle is nil")
 			continue
 		}
-		fmt.Println(*i.Status.SnapshotHandle)
+
+		if i.Labels == nil {
+			fmt.Println("VolumeSnapshotContents label is nil")
+			continue
+		}
+
 		if i.Labels["velero.io/backup-name"] == backupName {
 			tmp := strings.Split(*i.Status.SnapshotHandle, "/")
 			snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1])
@@ -120,13 +129,12 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba
 	return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace))
 }
 
-func CheckVolumeSnapshotCR(client TestClient, pods []string, nampespace, backupName string) error {
-	for _, podName := range pods {
-		if snapshotContentName, err := GetVolumeSnapshotContentNameByPod(client, podName, nampespace, backupName); err != nil || snapshotContentName == "" {
-			return errors.Wrap(err, "Fail to get Azure CSI snapshot content")
-		} else {
-			fmt.Println("Found volumesnapshotcontent: " + snapshotContentName)
-		}
+func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) error {
+	var err error
+	var snapshotContentNameList []string
+	if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil || len(snapshotContentNameList) != expectedCount {
+		return errors.Wrap(err, "Fail to get Azure CSI snapshot content")
 	}
+	fmt.Println(snapshotContentNameList)
 	return nil
 }

test/e2e/util/kibishii/kibishii_utils.go

@@ -28,7 +28,6 @@ import (
 
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
 	. "github.com/vmware-tanzu/velero/test/e2e"
-	util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
@@ -72,8 +71,13 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
 		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace)
 	}
 
-	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName,
-		kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil {
+	var BackupCfg BackupConfig
+	BackupCfg.BackupName = backupName
+	BackupCfg.Namespace = kibishiiNamespace
+	BackupCfg.BackupLocation = backupLocation
+	BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
+	BackupCfg.Selector = ""
+	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil {
 		RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
 		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
 	}
@@ -87,10 +91,6 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
 			if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
 				return errors.Wrapf(err, "Error waiting for uploads to complete")
 			}
-		} else if providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") {
-			if err := util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, kibishiiNamespace, backupName); err != nil {
-				return errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
-			}
 		}
 		snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList)
 		if err != nil {
@@ -108,14 +108,6 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
 	if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
 		return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
 	}
-	if useVolumeSnapshots && providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") {
-		err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
-			VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
-			backupName, snapshotCheckPoint)
-		if err != nil {
-			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
-		}
-	}
 	time.Sleep(5 * time.Minute)
 
 	// the snapshots of AWS may be still in pending status when do the restore, wait for a while
@@ -215,6 +207,7 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie
 	if err := WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil {
 		return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace)
 	}
+
 	// add the image pull secret to avoid the image pull limit issue of Docker Hub
 	if err := PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil {
 		return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace)

test/e2e/util/providers/common.go

@@ -37,10 +37,10 @@ type ObjectsInStorage interface {
 }
 
 func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
-	fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage [%s]", backupName, bslPrefix)
+	fmt.Printf("|| VERIFICATION || - %s %s should exist in storage [%s]\n", subPrefix, backupName, bslPrefix)
 	exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 	if !exist {
-		return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName))
+		return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected\n", backupName))
 	}
 	fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket)
 	return nil
@@ -48,11 +48,11 @@ func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bsl
 func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error {
 	var err error
 	var exist bool
-	fmt.Printf("|| VERIFICATION || - Backup %s should not exist in storage %s", backupName, bslPrefix)
+	fmt.Printf("|| VERIFICATION || - %s %s should not exist in storage %s\n", subPrefix, backupName, bslPrefix)
 	for i := 0; i < retryTimes; i++ {
 		exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 		if err != nil {
-			return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store", backupName)
+			return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store\n", backupName)
 		}
 		if !exist {
 			fmt.Printf("|| EXPECTED || - Backup %s is not in object store\n", backupName)
@@ -60,7 +60,7 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket,
 		}
 		time.Sleep(1 * time.Minute)
 	}
-	return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion", backupName))
+	return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion\n", backupName))
 }
 func getProvider(cloudProvider string) (ObjectsInStorage, error) {
 	var s ObjectsInStorage

test/e2e/util/velero/install.go

@@ -90,6 +90,8 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnaps
 	veleroInstallOptions.UseRestic = !useVolumeSnapshots
 	veleroInstallOptions.Image = veleroCfg.VeleroImage
 	veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
+	GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency)
+	veleroInstallOptions.GarbageCollectionFrequency = GCFrequency
 
 	err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{
 		InstallOptions:         veleroInstallOptions,
@@ -131,6 +133,7 @@ func configvSpherePlugin() error {
 	if err != nil {
 		return errors.WithMessagef(err, "Failed to create velero-vsphere-plugin-config configmap in %s namespace", VeleroCfg.VeleroNamespace)
 	}
+	fmt.Println("configvSpherePlugin: WaitForConfigMapComplete")
 	err = WaitForConfigMapComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Failed to ensure configmap %s completion in namespace: %s", configmaptName, VeleroCfg.VeleroNamespace))
@@ -209,6 +212,9 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
 
 		}
 	}
+	if options.GarbageCollectionFrequency > 0 {
+		args = append(args, fmt.Sprintf("--garbage-collection-frequency=%v", options.GarbageCollectionFrequency))
+	}
 
 	if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil {
 		return err

test/e2e/util/velero/velero_utils.go

@@ -47,6 +47,7 @@ import (
 )
 
 const BackupObjectsPrefix = "backups"
+const RestoreObjectsPrefix = "restores"
 const PluginsObjectsPrefix = "plugins"
 
 var pluginsMatrix = map[string]map[string][]string{
@@ -246,19 +247,18 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
 }
 
 // VeleroBackupNamespace uses the veleroCLI to backup a namespace.
-func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace, backupName, namespace, backupLocation string,
-	useVolumeSnapshots bool, selector string) error {
+func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace string, backupCfg BackupConfig) error {
 	args := []string{
 		"--namespace", veleroNamespace,
-		"create", "backup", backupName,
-		"--include-namespaces", namespace,
+		"create", "backup", backupCfg.BackupName,
+		"--include-namespaces", backupCfg.Namespace,
 		"--wait",
 	}
-	if selector != "" {
-		args = append(args, "--selector", selector)
+	if backupCfg.Selector != "" {
+		args = append(args, "--selector", backupCfg.Selector)
 	}
 
-	if useVolumeSnapshots {
+	if backupCfg.UseVolumeSnapshots {
 		args = append(args, "--snapshot-volumes")
 	} else {
 		args = append(args, "--default-volumes-to-restic")
@@ -268,11 +268,13 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace, back
 		// TODO This can be removed if the logic of vSphere plugin bump up to 1.3
 		args = append(args, "--snapshot-volumes=false")
 	}
-	if backupLocation != "" {
-		args = append(args, "--storage-location", backupLocation)
+	if backupCfg.BackupLocation != "" {
+		args = append(args, "--storage-location", backupCfg.BackupLocation)
 	}
-
-	return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
+	if backupCfg.TTL != 0 {
+		args = append(args, "--ttl", backupCfg.TTL.String())
+	}
+	return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupCfg.BackupName, args)
 }
 
 // VeleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace.
@@ -502,7 +504,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace
 	if err != nil {
 		fmt.Print(stdout)
 		fmt.Print(stderr)
-		return nil, errors.Wrap(err, "failed to verify")
+		return nil, errors.Wrap(err, "Failed to get vSphere snapshot ID list from snapshots.backupdriver.cnsdp.vmware.com")
 	}
 	stdout = strings.Replace(stdout, "'", "", -1)
 	lines := strings.Split(stdout, "\n")
@@ -814,9 +816,9 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCou
 	snapshotCheckPoint.ExpectCount = expectCount
 	snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
 	snapshotCheckPoint.PodName = kibishiiPodNameList
-	if strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
+	if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
 		snapshotCheckPoint.EnableCSI = true
-		if err := util.CheckVolumeSnapshotCR(client, kibishiiPodNameList, namespaceBackedUp, backupName); err != nil {
+		if err := util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil {
 			return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
 		}
 		var err error
@@ -832,3 +834,24 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCou
 	fmt.Println(snapshotCheckPoint)
 	return snapshotCheckPoint, nil
 }
+
+func GetBackupTTL(ctx context.Context, veleroNamespace, backupName string) (string, error) {
+
+	checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
+		"get", "backup", "-n", veleroNamespace, backupName, "-o=jsonpath='{.spec.ttl}'")
+	fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd)
+	stdout, stderr, err := veleroexec.RunCommand(checkSnapshotCmd)
+	if err != nil {
+		fmt.Print(stdout)
+		fmt.Print(stderr)
+		return "", errors.Wrap(err, "failed to verify")
+	}
+	// lines := strings.Split(stdout, "\n")
+	// complete := true
+	// for _, curLine := range lines {
+	// 	fmt.Println(curLine)
+
+	// }
+	// return complete, nil
+	return stdout, err
+}