Merge pull request #7176 from danfengliu/fix-issue-of-hiiting-snapshot-limit

Add sleep to avoid snapshot limitation issue

test/e2e/backup/backup.go

@@ -77,7 +77,13 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
 			Skip("Volume snapshots not supported on kind")
 		}
-
+		// [SKIP]: Static provisioning for vSphere CSI driver works differently from other drivers.
+		//         For vSphere CSI, after you create a PV specifying an existing volume handle, CSI
+		//         syncer will need to register it with CNS. For other CSI drivers, static provisioning
+		//         usually does not go through storage system at all.  That's probably why it took longer
+		if backupRestoreTestConfig.isRetainPVTest && veleroCfg.CloudProvider == "vsphere" {
+			Skip("Skip due to vSphere CSI driver long time issue of Static provisioning")
+		}
 		var err error
 		flag.Parse()
 		UUIDgen, err = uuid.NewRandom()
@@ -95,7 +101,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 				}
 				DeleteStorageClass(context.Background(), *veleroCfg.ClientToInstallVelero, KibishiiStorageClassName)
 			})
-			if veleroCfg.InstallVelero {
+			if InstallVelero {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 				defer ctxCancel()
 				err = VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)
@@ -106,7 +112,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 
 	When("kibishii is the sample workload", func() {
 		It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
-			if veleroCfg.InstallVelero {
+			if InstallVelero {
 				if useVolumeSnapshots {
 					//Install node agent also
 					veleroCfg.UseNodeAgent = useVolumeSnapshots
@@ -147,7 +153,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 			if veleroCfg.AdditionalBSLCredentials == "" {
 				Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
 			}
-			if veleroCfg.InstallVelero {
+			if InstallVelero {
 				if useVolumeSnapshots {
 					veleroCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
 				} else { //FS volume backup

test/e2e/backups/deletion.go

@@ -59,8 +59,8 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 		}
 		var err error
 		flag.Parse()
-		if veleroCfg.InstallVelero {
-			Expect(PrepareVelero(context.Background(), "backup deletion")).To(Succeed())
+		if InstallVelero {
+			Expect(PrepareVelero(context.Background(), "backup deletion", veleroCfg)).To(Succeed())
 		}
 		UUIDgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
@@ -178,6 +178,11 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
 		}
 	}
+
+	// Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded
+	// Sleep for more than 15 seconds to avoid this issue.
+	time.Sleep(1 * time.Minute)
+
 	backupName = "backup-1-" + UUIDgen.String()
 	BackupCfg.BackupName = backupName
 

test/e2e/backups/sync_backups.go

@@ -56,7 +56,7 @@ func BackupsSyncTest() {
 
 	BeforeEach(func() {
 		flag.Parse()
-		if VeleroCfg.InstallVelero {
+		if InstallVelero {
 			veleroCfg := VeleroCfg
 			veleroCfg.UseVolumeSnapshots = false
 			Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
@@ -68,7 +68,7 @@ func BackupsSyncTest() {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), *VeleroCfg.ClientToInstallVelero)
 			})
-			if VeleroCfg.InstallVelero {
+			if InstallVelero {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 				defer ctxCancel()
 				Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())

test/e2e/backups/ttl.go

@@ -32,7 +32,6 @@ import (
 
 	. "github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
-
 	. "github.com/vmware-tanzu/velero/test/util/kibishii"
 	. "github.com/vmware-tanzu/velero/test/util/providers"
 	. "github.com/vmware-tanzu/velero/test/util/velero"
@@ -66,7 +65,7 @@ func TTLTest() {
 	BeforeEach(func() {
 		flag.Parse()
 		veleroCfg = VeleroCfg
-		if veleroCfg.InstallVelero {
+		if InstallVelero {
 			// Make sure GCFrequency is shorter than backup TTL
 			veleroCfg.GCFrequency = "4m0s"
 			veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
@@ -82,7 +81,7 @@ func TTLTest() {
 			})
 			ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 			defer ctxCancel()
-			if veleroCfg.InstallVelero {
+			if InstallVelero {
 				Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
 			}
 			Expect(DeleteNamespace(ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))

test/e2e/basic/api-group/enable_api_group_extentions.go

@@ -71,7 +71,7 @@ func APIExtensionsVersionsTest() {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), *veleroCfg.DefaultClient)
 			})
-			if veleroCfg.InstallVelero {
+			if InstallVelero {
 				By("Uninstall Velero and delete CRD ", func() {
 					ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 					defer ctxCancel()

test/e2e/basic/api-group/enable_api_group_versions.go

@@ -69,7 +69,7 @@ func APIGropuVersionsTest() {
 		flag.Parse()
 		// TODO: install Velero once for the test suite once feature flag is
 		// removed and velero installation becomes the same as other e2e tests.
-		if veleroCfg.InstallVelero {
+		if InstallVelero {
 			veleroCfg.Features = "EnableAPIGroupVersions"
 			veleroCfg.UseVolumeSnapshots = false
 			err = VeleroInstall(context.Background(), &veleroCfg, false)
@@ -96,7 +96,7 @@ func APIGropuVersionsTest() {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
 			})
-			if veleroCfg.InstallVelero {
+			if InstallVelero {
 				By("Uninstall Velero in api group version case", func() {
 					Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred())
 				})

test/e2e/basic/namespace-mapping.go

@@ -64,7 +64,7 @@ func (n *NamespaceMapping) Init() error {
 	n.MappedNamespaceList = mappedNSList
 	fmt.Println(mappedNSList)
 	n.BackupArgs = []string{
-		"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", n.BackupName,
+		"create", "--namespace", n.VeleroCfg.VeleroNamespace, "backup", n.BackupName,
 		"--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait",
 	}
 	if n.UseVolumeSnapshots {
@@ -74,7 +74,7 @@ func (n *NamespaceMapping) Init() error {
 		n.BackupArgs = append(n.BackupArgs, "--default-volumes-to-fs-backup")
 	}
 	n.RestoreArgs = []string{
-		"create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName,
+		"create", "--namespace", n.VeleroCfg.VeleroNamespace, "restore", n.RestoreName,
 		"--from-backup", n.BackupName, "--namespace-mappings", mappedNS,
 		"--wait",
 	}
@@ -89,9 +89,9 @@ func (n *NamespaceMapping) CreateResources() error {
 			Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns))
 		})
 		By("Deploy sample workload of Kibishii", func() {
-			Expect(KibishiiPrepareBeforeBackup(n.Ctx, n.Client, VeleroCfg.CloudProvider,
-				ns, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features,
-				VeleroCfg.KibishiiDirectory, false, n.kibishiiData)).To(Succeed())
+			Expect(KibishiiPrepareBeforeBackup(n.Ctx, n.Client, n.VeleroCfg.CloudProvider,
+				ns, n.VeleroCfg.RegistryCredentialFile, n.VeleroCfg.Features,
+				n.VeleroCfg.KibishiiDirectory, false, n.kibishiiData)).To(Succeed())
 		})
 	}
 	return nil

test/e2e/bsl-mgmt/deletion.go

@@ -30,7 +30,6 @@ import (
 	. "github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/util/kibishii"
-
 	. "github.com/vmware-tanzu/velero/test/util/providers"
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
@@ -69,8 +68,8 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 		flag.Parse()
 		UUIDgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
-		if veleroCfg.InstallVelero {
-			Expect(PrepareVelero(context.Background(), "BSL Deletion")).To(Succeed())
+		if InstallVelero {
+			Expect(PrepareVelero(context.Background(), "BSL Deletion", veleroCfg)).To(Succeed())
 		}
 	})
 

test/e2e/e2e_suite_test.go

@@ -50,7 +50,7 @@ import (
 )
 
 func init() {
-	VeleroCfg.Options = &install.Options{}
+	VeleroCfg.Options = install.Options{}
 	flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into.  Required.")
 	flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
 	flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
@@ -69,7 +69,7 @@ func init() {
 	flag.StringVar(&VeleroCfg.BSLPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.")
 	flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
 	flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into")
-	flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test.  Optional.")
+	flag.BoolVar(&InstallVelero, "install-velero", true, "install/uninstall velero during the test.  Optional.")
 	flag.BoolVar(&VeleroCfg.UseNodeAgent, "use-node-agent", true, "whether deploy node agent daemonset velero during the test.  Optional.")
 	flag.BoolVar(&VeleroCfg.UseVolumeSnapshots, "use-volume-snapshots", true, "whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.")
 	flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
@@ -209,14 +209,14 @@ func TestE2e(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	if VeleroCfg.InstallVelero {
+	if InstallVelero {
 		By("Install test resources before testing")
-		Expect(PrepareVelero(context.Background(), "install resource before testing")).To(Succeed())
+		Expect(PrepareVelero(context.Background(), "install resource before testing", VeleroCfg)).To(Succeed())
 	}
 })
 
 var _ = AfterSuite(func() {
-	if VeleroCfg.InstallVelero && !VeleroCfg.Debug {
+	if InstallVelero && !VeleroCfg.Debug {
 		By("release test resources after testing")
 		ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 		defer ctxCancel()

test/e2e/migration/migration.go

@@ -63,15 +63,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
 			Skip("Volume snapshots not supported on kind")
 		}
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" && !veleroCfg.SnapshotMoveData {
-			Skip("Volume snapshots migration not supported on AWS provisioned by Sheperd public pool")
-		}
 
 		if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
 			Skip("Migration test needs 2 clusters")
 		}
 		// need to uninstall Velero first in case of the affection of the existing global velero installation
-		if veleroCfg.InstallVelero {
+		if InstallVelero {
 			By("Uninstall Velero", func() {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 				defer ctxCancel()
@@ -100,7 +97,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 				DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
 			})
 
-			if veleroCfg.InstallVelero {
+			if InstallVelero {
 				By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() {
 					DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
 				})
@@ -258,7 +255,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 			}
 
 			if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" &&
-				strings.EqualFold(veleroCfg.Features, "EnableCSI") &&
+				strings.EqualFold(veleroCfg.Features, FeatureCSI) &&
 				!OriginVeleroCfg.SnapshotMoveData {
 				By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
 					time.Sleep(5 * time.Minute)

test/e2e/privilegesmgmt/ssr.go

@@ -43,8 +43,8 @@ func SSRTest() {
 	BeforeEach(func() {
 		flag.Parse()
 		veleroCfg.UseVolumeSnapshots = false
-		if veleroCfg.InstallVelero {
-			Expect(PrepareVelero(context.Background(), "SSR test")).To(Succeed())
+		if InstallVelero {
+			Expect(PrepareVelero(context.Background(), "SSR test", veleroCfg)).To(Succeed())
 		}
 	})
 

test/e2e/test/test.go

@@ -26,7 +26,6 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/pkg/errors"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -83,15 +82,15 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
 		BeforeEach(func() {
 			flag.Parse()
 			// Using the global velero config which covered the installation for most common cases
-			veleroCfg := VeleroCfg
+			veleroCfg := test.GetTestCase().VeleroCfg
 			// TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed
 			// TODO: Although this issue is closed, but it's not fixed.
 			// TODO: After bump up k8s version in AWS pipeline, this issue also apply for AWS pipeline.
 			if (veleroCfg.CloudProvider == "azure" || veleroCfg.CloudProvider == "aws") && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") {
 				Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS")
 			}
-			if veleroCfg.InstallVelero {
-				Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName)).To(Succeed())
+			if InstallVelero {
+				Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, veleroCfg)).To(Succeed())
 			}
 		})
 		It(test.GetTestMsg().Text, func() {
@@ -102,7 +101,6 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
 
 func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
 	return func() {
-		veleroCfg := VeleroCfg
 		for k := range tests {
 			Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc))
 			defer tests[k].GetTestCase().CtxCancel()
@@ -110,8 +108,8 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
 
 		BeforeEach(func() {
 			flag.Parse()
-			if veleroCfg.InstallVelero {
-				Expect(PrepareVelero(context.Background(), tests[0].GetTestCase().CaseBaseName)).To(Succeed())
+			if InstallVelero {
+				Expect(PrepareVelero(context.Background(), tests[0].GetTestCase().CaseBaseName, tests[0].GetTestCase().VeleroCfg)).To(Succeed())
 			}
 		})
 

test/e2e/upgrade/upgrade.go

@@ -61,7 +61,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 
 	BeforeEach(func() {
 		veleroCfg = VeleroCfg
-		if !veleroCfg.InstallVelero {
+		veleroCfg.IsUpgradeTest = true
+		if !InstallVelero {
 			Skip("Upgrade test should not be triggered if veleroCfg.InstallVelero is set to false")
 		}
 		if (len(veleroCfg.UpgradeFromVeleroVersion)) == 0 {
@@ -74,7 +75,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 			Skip("VeleroCLI should be provide")
 		}
 		// need to uninstall Velero first in case of the affection of the existing global velero installation
-		if veleroCfg.InstallVelero {
+		if InstallVelero {
 			By("Uninstall Velero", func() {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 				defer ctxCancel()
@@ -91,7 +92,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 			By(fmt.Sprintf("Delete sample workload namespace %s", upgradeNamespace), func() {
 				DeleteNamespace(context.Background(), *veleroCfg.ClientToInstallVelero, upgradeNamespace, true)
 			})
-			if veleroCfg.InstallVelero {
+			if InstallVelero {
 				By("Uninstall Velero", func() {
 					ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 					defer ctxCancel()
@@ -210,7 +211,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 					fmt.Sprintf("failed to delete namespace %s", upgradeNamespace))
 			})
 
-			if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, "EnableCSI") {
+			if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, FeatureCSI) {
 				// Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before
 				//   v1.0 have issues to fail upgrade case.
 				By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {

test/perf/e2e_suite_test.go

@@ -30,7 +30,6 @@ import (
 
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	. "github.com/vmware-tanzu/velero/test"
-
 	"github.com/vmware-tanzu/velero/test/perf/backup"
 	"github.com/vmware-tanzu/velero/test/perf/basic"
 	"github.com/vmware-tanzu/velero/test/perf/restore"
@@ -41,7 +40,7 @@ import (
 )
 
 func init() {
-	VeleroCfg.Options = &install.Options{}
+	VeleroCfg.Options = install.Options{}
 	flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into.  Required.")
 	flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
 	flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
@@ -56,7 +55,7 @@ func init() {
 	flag.StringVar(&VeleroCfg.BSLPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.")
 	flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
 	flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into")
-	flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test.  Optional.")
+	flag.BoolVar(&InstallVelero, "install-velero", true, "install/uninstall velero during the test.  Optional.")
 	flag.BoolVar(&VeleroCfg.UseNodeAgent, "use-node-agent", true, "whether deploy node agent daemonset velero during the test.  Optional.")
 	flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
 	flag.StringVar(&VeleroCfg.NodeAgentPodCPULimit, "node-agent-pod-cpu-limit", "4", "CPU limit for node agent pod. Optional.")
@@ -120,15 +119,15 @@ func TestE2e(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	if VeleroCfg.InstallVelero {
+	if InstallVelero {
 		By("Install test resources before testing BeforeSuite")
-		Expect(PrepareVelero(context.Background(), "install resource before testing")).To(Succeed())
+		Expect(PrepareVelero(context.Background(), "install resource before testing", VeleroCfg)).To(Succeed())
 	}
 })
 
 var _ = AfterSuite(func() {
 	Expect(report.GenerateYamlReport()).To(Succeed())
-	if VeleroCfg.InstallVelero && !VeleroCfg.Debug {
+	if InstallVelero && !VeleroCfg.Debug {
 		By("release test resources after testing")
 		Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
 	}

test/perf/test/test.go

@@ -24,16 +24,14 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/pkg/errors"
 
-	"github.com/vmware-tanzu/velero/test/util/report"
-	"github.com/vmware-tanzu/velero/test/util/velero"
-
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	. "github.com/vmware-tanzu/velero/test"
 	"github.com/vmware-tanzu/velero/test/perf/metrics"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
+	"github.com/vmware-tanzu/velero/test/util/report"
+	"github.com/vmware-tanzu/velero/test/util/velero"
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
 
@@ -86,8 +84,8 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
 		By(fmt.Sprintf("Run test %s ...... \n", test.GetTestCase().CaseBaseName))
 		BeforeEach(func() {
 			// Using the global velero config which covered the installation for most common cases
-			if VeleroCfg.InstallVelero {
-				Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName)).To(Succeed())
+			if InstallVelero {
+				Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, VeleroCfg)).To(Succeed())
 			}
 		})
 		It(test.GetTestMsg().Text, func() {

test/types.go

@@ -27,7 +27,9 @@ import (
 
 const StorageClassName = "e2e-storage-class"
 const StorageClassName2 = "e2e-storage-class-2"
+const FeatureCSI = "EnableCSI"
 
+var InstallVelero bool
 var UUIDgen uuid.UUID
 
 var VeleroCfg VeleroConfig
@@ -41,7 +43,7 @@ var ReportData *Report
 
 type VeleroConfig struct {
 	VeleroCfgInPerf
-	*install.Options
+	install.Options
 	VeleroCLI                         string
 	VeleroImage                       string
 	VeleroVersion                     string
@@ -66,7 +68,6 @@ type VeleroConfig struct {
 	MigrateFromVeleroCLI              string
 	Plugins                           string
 	AddBSLPlugins                     string
-	InstallVelero                     bool
 	KibishiiDirectory                 string
 	Debug                             bool
 	GCFrequency                       string
@@ -83,6 +84,7 @@ type VeleroConfig struct {
 	StandbyClusterPlugins             string
 	StandbyClusterOjbectStoreProvider string
 	DebugVeleroPodRestart             bool
+	IsUpgradeTest                     bool
 }
 
 type VeleroCfgInPerf struct {

test/util/csi/common.go

@@ -85,7 +85,7 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error
 	}
 
 	if len(snapshotHandleList) == 0 {
-		fmt.Printf("No VolumeSnapshotContent from backup %s", backupName)
+		fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName)
 	}
 	return snapshotHandleList, nil
 }
@@ -122,7 +122,7 @@ func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, err
 	}
 
 	if len(snapshotHandleList) == 0 {
-		fmt.Printf("No VolumeSnapshotContent from backup %s", backupName)
+		fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName)
 	}
 	return snapshotHandleList, nil
 }
@@ -179,8 +179,8 @@ func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount i
 		return nil, errors.New("API version is invalid")
 	}
 	if len(snapshotContentNameList) != expectedCount {
-		return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount))
+		return nil, errors.New(fmt.Sprintf("Snapshot content count %d is not as expect %d", len(snapshotContentNameList), expectedCount))
 	}
-	fmt.Println(snapshotContentNameList)
+	fmt.Printf("snapshotContentNameList: %v \n", snapshotContentNameList)
 	return snapshotContentNameList, nil
 }

test/util/kibishii/kibishii_utils.go

@@ -151,7 +151,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 		} else {
 			// wait for a period to confirm no snapshots exist for the backup
 			time.Sleep(5 * time.Minute)
-			if strings.EqualFold(veleroFeatures, "EnableCSI") {
+			if strings.EqualFold(veleroFeatures, FeatureCSI) {
 				_, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 0,
 					kibishiiNamespace, backupName, KibishiiPVCNameList)
 				if err != nil {
@@ -245,11 +245,11 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures,
 	kibishiiDirectory string, useVolumeSnapshots bool, workerReplicas int) error {
 	if strings.EqualFold(cloudPlatform, "azure") &&
-		strings.EqualFold(veleroFeatures, "EnableCSI") {
+		strings.EqualFold(veleroFeatures, FeatureCSI) {
 		cloudPlatform = "azure-csi"
 	}
 	if strings.EqualFold(cloudPlatform, "aws") &&
-		strings.EqualFold(veleroFeatures, "EnableCSI") {
+		strings.EqualFold(veleroFeatures, FeatureCSI) {
 		cloudPlatform = "aws-csi"
 	}
 	// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories

test/util/providers/aws_utils.go

@@ -25,16 +25,14 @@ import (
 	"os"
 	"strings"
 
-	"github.com/aws/aws-sdk-go-v2/config"
-	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
-
 	"github.com/aws/aws-sdk-go-v2/aws"
 	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+	"github.com/aws/aws-sdk-go-v2/config"
 	s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 
@@ -310,19 +308,27 @@ func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj
 	if region == "minio" {
 		return errors.New("No snapshot for Minio provider")
 	}
+
 	cfg, err := newAWSConfig(region, "", cloudCredentialsFile, false, "")
 	if err != nil {
 		return errors.Wrapf(err, "Failed to create AWS config of region %s", region)
 	}
+
 	ec2Client := ec2.NewFromConfig(cfg)
 	input := &ec2.DescribeSnapshotsInput{
 		OwnerIds: []string{"self"},
-		Filters: []ec2types.Filter{
-			{
-				Name:   aws.String("tag:velero.io/backup"),
-				Values: []string{backupObject},
+	}
+
+	if !snapshotCheck.EnableCSI {
+		input = &ec2.DescribeSnapshotsInput{
+			OwnerIds: []string{"self"},
+			Filters: []ec2types.Filter{
+				{
+					Name:   aws.String("tag:velero.io/backup"),
+					Values: []string{backupObject},
+				},
 			},
-		},
+		}
 	}
 
 	result, err := ec2Client.DescribeSnapshots(context.Background(), input)
@@ -330,21 +336,34 @@ func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj
 		fmt.Println(err)
 	}
 
-	for _, n := range result.Snapshots {
-		fmt.Println(n.SnapshotId)
-		if n.SnapshotId != nil {
-			fmt.Println(*n.SnapshotId)
+	var actualCount int
+	if snapshotCheck.EnableCSI {
+		for _, snapshotId := range snapshotCheck.SnapshotIDList {
+			for _, n := range result.Snapshots {
+				if n.SnapshotId != nil && (*n.SnapshotId == snapshotId) {
+					actualCount++
+					fmt.Printf("SnapshotId: %v, Tags: %v \n", *n.SnapshotId, n.Tags)
+					if n.VolumeId != nil {
+						fmt.Printf("VolumeId: %v \n", *n.VolumeId)
+					}
+				}
+			}
 		}
-		fmt.Println(n.Tags)
-		fmt.Println(n.VolumeId)
-		if n.VolumeId != nil {
-			fmt.Println(*n.VolumeId)
-		}
-	}
-	if len(result.Snapshots) != snapshotCheck.ExpectCount {
-		return errors.New(fmt.Sprintf("Snapshot count is not as expected %d", snapshotCheck.ExpectCount))
 	} else {
-		fmt.Printf("Snapshot count %d is as expected %d\n", len(result.Snapshots), snapshotCheck.ExpectCount)
+		for _, n := range result.Snapshots {
+			if n.SnapshotId != nil {
+				fmt.Printf("SnapshotId: %v, Tags: %v \n", *n.SnapshotId, n.Tags)
+				if n.VolumeId != nil {
+					fmt.Printf("VolumeId: %v \n", *n.VolumeId)
+				}
+			}
+		}
+		actualCount = len(result.Snapshots)
+	}
+	if actualCount != snapshotCheck.ExpectCount {
+		return errors.New(fmt.Sprintf("Snapshot count %d is not as expected %d", actualCount, snapshotCheck.ExpectCount))
+	} else {
+		fmt.Printf("Snapshot count %d is as expected %d\n", actualCount, snapshotCheck.ExpectCount)
 		return nil
 	}
 }

test/util/providers/common.go

@@ -168,7 +168,7 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig
 	} else {
 		err = s.IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName, snapshotCheck)
 		if err != nil {
-			return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup%s", backupName))
+			return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup %s", backupName))
 		}
 	}
 	return nil

test/util/velero/install.go

@@ -57,6 +57,7 @@ type installOptions struct {
 
 func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluster bool) error {
 	fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05"))
+
 	// veleroCfg struct including a set of BSL params and a set of additional BSL params,
 	// additional BSL set is for additional BSL test only, so only default BSL set is effective
 	// for VeleroInstall().
@@ -247,7 +248,7 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 	fmt.Println("Start to install Azure VolumeSnapshotClass ...")
 	if len(options.Features) > 0 {
 		args = append(args, "--features", options.Features)
-		if strings.EqualFold(options.Features, "EnableCSI") && options.UseVolumeSnapshots {
+		if strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
 			if strings.EqualFold(cloudProvider, "azure") {
 				if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
 					return err
@@ -578,13 +579,13 @@ func IsVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) (bo
 	return true, nil
 }
 
-func PrepareVelero(ctx context.Context, caseName string) error {
-	ready, err := IsVeleroReady(context.Background(), VeleroCfg.VeleroNamespace, VeleroCfg.UseNodeAgent)
+func PrepareVelero(ctx context.Context, caseName string, veleroCfg VeleroConfig) error {
+	ready, err := IsVeleroReady(context.Background(), veleroCfg.VeleroNamespace, veleroCfg.UseNodeAgent)
 	if err != nil {
 		fmt.Printf("error in checking velero status with %v", err)
 		ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 		defer ctxCancel()
-		VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
+		VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)
 		ready = false
 	}
 	if ready {
@@ -592,7 +593,7 @@ func PrepareVelero(ctx context.Context, caseName string) error {
 		return nil
 	}
 	fmt.Printf("need to install velero for case %s \n", caseName)
-	return VeleroInstall(context.Background(), &VeleroCfg, false)
+	return VeleroInstall(context.Background(), &veleroCfg, false)
 }
 
 func VeleroUninstall(ctx context.Context, cli, namespace string) error {

test/util/velero/velero_utils.go

@@ -141,7 +141,7 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature st
 		plugins = append(plugins, pluginsForObjectStoreProvider...)
 	}
 
-	if strings.EqualFold(feature, "EnableCSI") {
+	if strings.EqualFold(feature, FeatureCSI) {
 		pluginsForFeature, ok = cloudMap["csi"]
 		if !ok {
 			return nil, errors.Errorf("fail to get CSI plugins by version: %s ", version)
@@ -640,7 +640,7 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) {
 				return nil, errors.WithMessage(err, "failed to get velero version")
 			}
 		}
-		if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" {
+		if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" && !veleroCfg.IsUpgradeTest {
 			needDataMoverPlugin = true
 		}
 		plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature, needDataMoverPlugin)
@@ -1199,7 +1199,7 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCoun
 	snapshotCheckPoint.ExpectCount = expectCount
 	snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
 	snapshotCheckPoint.PodName = KibishiiPVCNameList
-	if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
+	if (VeleroCfg.CloudProvider == "azure" || VeleroCfg.CloudProvider == "aws") && strings.EqualFold(VeleroCfg.Features, FeatureCSI) {
 		snapshotCheckPoint.EnableCSI = true
 		resourceName := "snapshot.storage.k8s.io"
 
@@ -1215,7 +1215,7 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCoun
 			return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
 		}
 	}
-	fmt.Println(snapshotCheckPoint)
+	fmt.Printf("snapshotCheckPoint: %v \n", snapshotCheckPoint)
 	return snapshotCheckPoint, nil
 }